From f43bfd69e97408d06a5de4851aec77f8754bd72f Mon Sep 17 00:00:00 2001 From: lambda <1wei@live.com> Date: Fri, 21 Jul 2023 15:38:32 +0800 Subject: [PATCH] GH-36770: [C++] Use custom endpoint for s3 using environment variable AWS_ENDPOINT_URL (#36791) ### Rationale for this change we need a way to read custom object storage (such as minio host or other s3-like storage). use environment variable `AWS_ENDPOINT_URL ` ### What changes are included in this PR? set variable endpoint_override according the environment variable ### Are these changes tested? unittest and tested on pyarrow ### Are there any user-facing changes? No * Closes: #36770 Authored-by: yiwei.wang Signed-off-by: Sutou Kouhei --- cpp/src/arrow/filesystem/s3fs.cc | 5 +++++ cpp/src/arrow/filesystem/s3fs_test.cc | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index c57fc7f291c68..29b45e1dc9abe 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -127,6 +127,7 @@ using internal::ToAwsString; using internal::ToURLEncodedAwsString; static const char kSep = '/'; +constexpr char kAwsEndpointUrlEnvVar[] = "AWS_ENDPOINT_URL"; // ----------------------------------------------------------------------- // S3ProxyOptions implementation @@ -337,6 +338,10 @@ Result S3Options::FromUri(const Uri& uri, std::string* out_path) { } else { options.ConfigureDefaultCredentials(); } + auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar); + if (endpoint_env.ok()) { + options.endpoint_override = *endpoint_env; + } bool region_set = false; for (const auto& kv : options_map) { diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 1426fe324b720..718304abaed63 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -297,6 +297,13 @@ TEST_F(S3OptionsTest, FromUri) { // Invalid option ASSERT_RAISES(Invalid, S3Options::FromUri("s3://mybucket/?xxx=zzz", &path)); + + // Endpoint from environment variable + { + EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000"); + ASSERT_OK_AND_ASSIGN(options, S3Options::FromUri("s3://mybucket/", &path)); + ASSERT_EQ(options.endpoint_override, "http://127.0.0.1:9000"); + } } TEST_F(S3OptionsTest, FromAccessKey) {