From 3150ca3ca929e37c8dc2158469d10060d4f74e7e Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 18 Dec 2019 16:03:10 +0900 Subject: [PATCH 1/9] add a method to get client instance only for table:import requests --- lib/td/command/common.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/td/command/common.rb b/lib/td/command/common.rb index fad5ff01..e3a6cdb3 100644 --- a/lib/td/command/common.rb +++ b/lib/td/command/common.rb @@ -47,7 +47,7 @@ def get_client(opts={}) # optional, if not provided a default is used from the ruby client library begin - if Config.endpoint + if !opts[:endpoint] && Config.endpoint opts[:endpoint] = Config.endpoint end rescue ConfigNotFoundError => e @@ -67,6 +67,12 @@ def get_client(opts={}) Client.new(apikey, opts) end + DEFAULT_IMPORT_ENDPOINT = "https://" + TreasureData::API::DEFAULT_IMPORT_ENDPOINT + + def get_import_client + get_client(endpoint: Config.import_endpoint || DEFAULT_IMPORT_ENDPOINT) + end + def get_ssl_client(opts={}) opts[:ssl] = true get_client(opts) From fd7fe8ff72dce1dc35374dfbaef0ada779a2abf0 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 18 Dec 2019 16:04:08 +0900 Subject: [PATCH 2/9] add a config parameter and command line option to specify api-import endpoint --- lib/td/command/runner.rb | 14 +++++++++++++- lib/td/config.rb | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/td/command/runner.rb b/lib/td/command/runner.rb index 99d88223..2fbcd225 100644 --- a/lib/td/command/runner.rb +++ b/lib/td/command/runner.rb @@ -6,11 +6,12 @@ def initialize @config_path = nil @apikey = nil @endpoint = nil + @import_endpoint = nil @prog_name = nil @insecure = false end - attr_accessor :apikey, :endpoint, :config_path, :prog_name, :insecure + attr_accessor :apikey, :endpoint, :import_endpoint, :config_path, :prog_name, :insecure def run(argv=ARGV) require 'td/version' @@ -73,6 +74,7 @@ def run(argv=ARGV) config_path = @config_path apikey = @apikey endpoint = @endpoint + import_endpoint = @import_endpoint || @endpoint insecure = nil $verbose = false #$debug = false @@ -94,6 +96,12 @@ def run(argv=ARGV) endpoint = e } + op.on('-E', '--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e| + require 'td/command/common' + Command.validate_api_endpoint(e) + import_endpoint = e + } + op.on('--insecure', "Insecure access: disable SSL (enabled by default)") {|b| insecure = true } @@ -140,6 +148,10 @@ def run(argv=ARGV) Config.endpoint = endpoint Config.cl_endpoint = true end + if import_endpoint + Config.endpoint = endpoint + Config.cl_endpoint = true + end if insecure Config.secure = false end diff --git a/lib/td/config.rb b/lib/td/config.rb index eead949b..2ce14d28 100644 --- a/lib/td/config.rb +++ b/lib/td/config.rb @@ -21,6 +21,9 @@ class Config @@endpoint = ENV['TREASURE_DATA_API_SERVER'] || ENV['TD_API_SERVER'] @@endpoint = nil if @@endpoint == "" @@cl_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line + @@import_endpoint = ENV['TREASURE_DATA_API_IMPORT_SERVER'] || ENV['TD_API_IMPORT_SERVER'] + @@import_endpoint = nil if @@endpoint == "" + @@cl_import_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line option @@secure = true @@retry_post_requests = false @@ -164,6 +167,22 @@ def self.cl_endpoint=(flag) @@cl_endpoint = flag end + def self.import_endpoint + @@import_endpoint || Config.read['account.import_endpoint'] + end + + def self.import_endpoint=(endpoint) + @@import_endpoint = endpoint + end + + def self.cl_import_endpoint + @@cl_import_endpoint + end + + def self.cl_import_endpoint=(flag) + @@cl_import_endpoint = flag + end + def self.workflow_endpoint case self.endpoint_domain when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?(eu01\.)?treasuredata\.(com|co\.jp)\z/i @@ -180,6 +199,8 @@ def self.cl_options_string string += " " unless string.empty? string += "-e #{@@endpoint}" if @@cl_endpoint string += " " unless string.empty? + string += "-E #{@@import_endpoint}" if @@cl_import_endpoint + string += " " unless string.empty? string end From 8f038713e9c9344e4c1981b5dc241b4233ec2683 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 18 Dec 2019 16:04:32 +0900 Subject: [PATCH 3/9] use a client instance for api-import only on import requests --- lib/td/command/table.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/td/command/table.rb b/lib/td/command/table.rb index 25747dab..2d7823bd 100644 --- a/lib/td/command/table.rb +++ b/lib/td/command/table.rb @@ -1,5 +1,6 @@ require 'td/helpers' require 'td/command/job' +require 'td/client/api' module TreasureData module Command @@ -453,17 +454,18 @@ def table_import(op) import_params[:table_name] = table_name import_params[:paths] = paths - client = get_client + api_client = get_client + import_client = get_import_client if auto_create - create_database_and_table_if_not_exist(client, db_name, table_name) + create_database_and_table_if_not_exist(api_client, db_name, table_name) end - do_table_import(client, import_params) + do_table_import(api_client, import_client, import_params) end private - def do_table_import(client, import_params) + def do_table_import(api_client, import_client, import_params) case import_params[:format] when 'json', 'msgpack' #unless time_key @@ -488,7 +490,7 @@ def do_table_import(client, import_params) end begin - db = client.database(import_params[:db_name]) + db = api_client.database(import_params[:db_name]) rescue ForbiddenError => e $stdout.puts "Warning: database and table validation skipped - #{e.message}" else @@ -521,7 +523,7 @@ def do_table_import(client, import_params) #require 'thread' files.zip(import_params[:paths]).each {|file, path| - import_log_file(file, path, client, import_params[:db_name], import_params[:table_name], parser) + import_log_file(file, path, import_client, import_params[:db_name], import_params[:table_name], parser) } $stdout.puts "done." From 1b9d06c093902b244eeb3cca0cc4a94bd6233678 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 18 Dec 2019 16:04:48 +0900 Subject: [PATCH 4/9] update the spec about table:import command --- spec/td/command/table_spec.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/td/command/table_spec.rb b/spec/td/command/table_spec.rb index 83a90b75..5e28206e 100644 --- a/spec/td/command/table_spec.rb +++ b/spec/td/command/table_spec.rb @@ -302,6 +302,7 @@ module TreasureData::Command let(:db_name) { 'database' } let(:table_name) { 'table' } let(:client) { double('client') } + let(:import_client) { double('import-client') } let(:command) { Class.new { include TreasureData::Command }.new } describe 'auto create table' do @@ -336,6 +337,7 @@ module TreasureData::Command describe 'time key' do before do allow(command).to receive(:get_client) { client } + allow(command).to receive(:get_import_client) { import_client } allow(command).to receive(:do_table_import) end let(:input_params) {{ @@ -355,7 +357,7 @@ module TreasureData::Command end it "with '#{tk_option}' option" do - expect(command).to receive(:do_table_import).with(client, input_params) + expect(command).to receive(:do_table_import).with(client, import_client, input_params) command.table_import(option) end end @@ -371,7 +373,7 @@ module TreasureData::Command end it 'without \'-t / --time-key\' option' do - expect(command).to receive(:do_table_import).with(client, input_params) + expect(command).to receive(:do_table_import).with(client, import_client, input_params) command.table_import(option) end end From beab44de03728a19ceaf918e83fa2079158ec940 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 18 Dec 2019 16:38:34 +0900 Subject: [PATCH 5/9] fix to rescue exceptions --- lib/td/command/common.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/td/command/common.rb b/lib/td/command/common.rb index e3a6cdb3..45333680 100644 --- a/lib/td/command/common.rb +++ b/lib/td/command/common.rb @@ -70,7 +70,12 @@ def get_client(opts={}) DEFAULT_IMPORT_ENDPOINT = "https://" + TreasureData::API::DEFAULT_IMPORT_ENDPOINT def get_import_client - get_client(endpoint: Config.import_endpoint || DEFAULT_IMPORT_ENDPOINT) + import_endpoint = begin + Config.import_endpoint || DEFAULT_IMPORT_ENDPOINT + rescue TreasureData::ConfigNotFoundError + DEFAULT_IMPORT_ENDPOINT + end + get_client(endpoint: import_endpoint) end def get_ssl_client(opts={}) From 20d0239b1d64c22bc9dd1084a5502c6aac9f07a3 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 19 Dec 2019 15:08:50 +0900 Subject: [PATCH 6/9] fix wrong setter --- lib/td/command/runner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/td/command/runner.rb b/lib/td/command/runner.rb index 2fbcd225..223ad9ab 100644 --- a/lib/td/command/runner.rb +++ b/lib/td/command/runner.rb @@ -149,8 +149,8 @@ def run(argv=ARGV) Config.cl_endpoint = true end if import_endpoint - Config.endpoint = endpoint - Config.cl_endpoint = true + Config.import_endpoint = endpoint + Config.cl_import_endpoint = true end if insecure Config.secure = false From dd94bd7b00990b23c22577a67ce72d5f7b599cf6 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 19 Dec 2019 16:45:56 +0900 Subject: [PATCH 7/9] change the command line option to use -i for import --- lib/td/command/runner.rb | 2 +- lib/td/config.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/td/command/runner.rb b/lib/td/command/runner.rb index 223ad9ab..eb6e39f4 100644 --- a/lib/td/command/runner.rb +++ b/lib/td/command/runner.rb @@ -96,7 +96,7 @@ def run(argv=ARGV) endpoint = e } - op.on('-E', '--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e| + op.on('-i', '--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e| require 'td/command/common' Command.validate_api_endpoint(e) import_endpoint = e diff --git a/lib/td/config.rb b/lib/td/config.rb index 2ce14d28..9e020c01 100644 --- a/lib/td/config.rb +++ b/lib/td/config.rb @@ -199,7 +199,7 @@ def self.cl_options_string string += " " unless string.empty? string += "-e #{@@endpoint}" if @@cl_endpoint string += " " unless string.empty? - string += "-E #{@@import_endpoint}" if @@cl_import_endpoint + string += "-i #{@@import_endpoint}" if @@cl_import_endpoint string += " " unless string.empty? string end From 7d0acb47a4ca9dd20a37d617191f02bc6443ccf3 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 19 Dec 2019 16:59:54 +0900 Subject: [PATCH 8/9] remove 1-char option --- lib/td/command/runner.rb | 2 +- lib/td/config.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/td/command/runner.rb b/lib/td/command/runner.rb index eb6e39f4..4f403066 100644 --- a/lib/td/command/runner.rb +++ b/lib/td/command/runner.rb @@ -96,7 +96,7 @@ def run(argv=ARGV) endpoint = e } - op.on('-i', '--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e| + op.on('--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e| require 'td/command/common' Command.validate_api_endpoint(e) import_endpoint = e diff --git a/lib/td/config.rb b/lib/td/config.rb index 9e020c01..fac39cb2 100644 --- a/lib/td/config.rb +++ b/lib/td/config.rb @@ -199,7 +199,7 @@ def self.cl_options_string string += " " unless string.empty? string += "-e #{@@endpoint}" if @@cl_endpoint string += " " unless string.empty? - string += "-i #{@@import_endpoint}" if @@cl_import_endpoint + string += "--import-endpoint #{@@import_endpoint}" if @@cl_import_endpoint string += " " unless string.empty? string end From 34c7b4c6f96965607b4dca7dbafa207d836235c9 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 19 Dec 2019 17:00:30 +0900 Subject: [PATCH 9/9] fix the bug to add 3 spaces even when just one option is enabled --- lib/td/config.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/td/config.rb b/lib/td/config.rb index fac39cb2..6ab1b4c2 100644 --- a/lib/td/config.rb +++ b/lib/td/config.rb @@ -195,12 +195,9 @@ def self.workflow_endpoint # renders the apikey and endpoint options as a string for the helper commands def self.cl_options_string string = "" - string += "-k #{@@apikey}" if @@cl_apikey - string += " " unless string.empty? - string += "-e #{@@endpoint}" if @@cl_endpoint - string += " " unless string.empty? - string += "--import-endpoint #{@@import_endpoint}" if @@cl_import_endpoint - string += " " unless string.empty? + string += "-k #{@@apikey} " if @@cl_apikey + string += "-e #{@@endpoint} " if @@cl_endpoint + string += "--import-endpoint #{@@import_endpoint} " if @@cl_import_endpoint string end