From adfe755c0bd4377255c1566813e98497d9700616 Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Wed, 31 Jul 2024 09:50:35 +0200 Subject: [PATCH 1/3] SNOW-1561613: Gather more data from failing Jenkins tests --- .../client/jdbc/cloud/storage/SnowflakeGCSClient.java | 3 ++- .../net/snowflake/client/jdbc/SnowflakeDriverIT.java | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java index d907973ac..cf78dafa6 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java @@ -993,12 +993,13 @@ private void uploadWithPresignedUrl( SqlState.INTERNAL_ERROR, "Unexpected: upload presigned URL invalid"); } catch (Exception e) { + e.printStackTrace(); throw new SnowflakeSQLLoggedException( queryId, session, ErrorCode.INTERNAL_ERROR.getMessageCode(), SqlState.INTERNAL_ERROR, - "Unexpected: upload with presigned url failed"); + "Unexpected: upload with presigned url failed: " + e.getMessage()); } } diff --git a/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java b/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java index a66dd4c4a..ce0f39840 100644 --- a/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java +++ b/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java @@ -1038,16 +1038,17 @@ static void findFile(Statement statement, String checkSQL) throws Throwable { if (resultSet.next()) { fileFound = true; + + // assert the first column not null + assertNotNull("Null result", resultSet.getString(1)); + break; } // give enough time for s3 eventual consistency for US region Thread.sleep(1000); - assertTrue("Could not find a file", fileFound); - - // assert the first column not null - assertNotNull("Null result", resultSet.getString(1)); } } + assertTrue("Could not find a file", fileFound); } @Test From 43ac8f045fc6ca55e58b4ea5bd614896696b15f7 Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Wed, 31 Jul 2024 14:00:54 +0200 Subject: [PATCH 2/3] SNOW-1561613: Log GCS digest --- .../snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java index cf78dafa6..f481653ec 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java @@ -1280,6 +1280,7 @@ private AbstractMap.SimpleEntry parseEncryptionData( @Override public void addDigestMetadata(StorageObjectMetadata meta, String digest) { if (!SnowflakeUtil.isBlank(digest)) { + System.out.println("Setting digest to '" + digest + "'"); meta.addUserMetadata("sfc-digest", digest); } } From 7d8e84cea817d2a603bdca41a284dcd7bb39db32 Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Wed, 31 Jul 2024 14:19:17 +0200 Subject: [PATCH 3/3] SNOW-1561613: Move failing OldDriver tests with GCP to LatestIT --- .../client/jdbc/SnowflakeDriverIT.java | 325 ----------------- .../client/jdbc/SnowflakeDriverLatestIT.java | 327 ++++++++++++++++++ 2 files changed, 327 insertions(+), 325 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java b/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java index ce0f39840..19d72bb0d 100644 --- a/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java +++ b/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java @@ -733,61 +733,6 @@ public void testDBMetadata() throws Throwable { } } - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testPutWithWildcardGCP() throws Throwable { - Properties _connectionProperties = new Properties(); - _connectionProperties.put("inject_wait_in_put", 5); - _connectionProperties.put("ssl", "off"); - try (Connection connection = - getConnection( - DONT_INJECT_SOCKET_TIMEOUT, _connectionProperties, false, false, "gcpaccount"); - Statement statement = connection.createStatement()) { - try { - String sourceFilePath = getFullPathFileInResource(TEST_DATA_FILE); - // replace file name with wildcard character - sourceFilePath = sourceFilePath.replace("orders_100.csv", "orders_10*.csv"); - - File destFolder = tmpFolder.newFolder(); - String destFolderCanonicalPath = destFolder.getCanonicalPath(); - String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; - statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); - statement.execute("CREATE OR REPLACE STAGE wildcard_stage"); - assertTrue( - "Failed to put a file", - statement.execute("PUT file://" + sourceFilePath + " @wildcard_stage")); - - findFile(statement, "ls @wildcard_stage/"); - - assertTrue( - "Failed to get files", - statement.execute( - "GET @wildcard_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); - - File downloaded; - // download the files we just uploaded to stage - for (int i = 0; i < fileNames.length; i++) { - // Make sure that the downloaded file exists, it should be gzip compressed - downloaded = new File(destFolderCanonicalPathWithSeparator + fileNames[i] + ".gz"); - assertTrue(downloaded.exists()); - - Process p = - Runtime.getRuntime() - .exec("gzip -d " + destFolderCanonicalPathWithSeparator + fileNames[i] + ".gz"); - p.waitFor(); - - String individualFilePath = sourceFilePath.replace("orders_10*.csv", fileNames[i]); - - File original = new File(individualFilePath); - File unzipped = new File(destFolderCanonicalPathWithSeparator + fileNames[i]); - assertEquals(original.length(), unzipped.length()); - assertTrue(FileUtils.contentEquals(original, unzipped)); - } - } finally { - statement.execute("DROP STAGE IF EXISTS wildcard_stage"); - } - } - } /** * helper function for creating large file in Java. Copies info from 1 file to another @@ -807,227 +752,6 @@ private void copyContentFrom(File file1, File file2) throws Exception { } } - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testPutGetLargeFileGCP() throws Throwable { - try (Connection connection = getConnection("gcpaccount"); - Statement statement = connection.createStatement()) { - try { - File destFolder = tmpFolder.newFolder(); - String destFolderCanonicalPath = destFolder.getCanonicalPath(); - String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; - - File largeTempFile = tmpFolder.newFile("largeFile.csv"); - try (BufferedWriter bw = new BufferedWriter(new FileWriter(largeTempFile))) { - bw.write("Creating large test file for GCP PUT/GET test"); - bw.write(System.lineSeparator()); - bw.write("Creating large test file for GCP PUT/GET test"); - bw.write(System.lineSeparator()); - } - File largeTempFile2 = tmpFolder.newFile("largeFile2.csv"); - - String sourceFilePath = largeTempFile.getCanonicalPath(); - - // copy info from 1 file to another and continue doubling file size until we reach ~1.5GB, - // which is a large file - for (int i = 0; i < 12; i++) { - copyContentFrom(largeTempFile, largeTempFile2); - copyContentFrom(largeTempFile2, largeTempFile); - } - - statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); - - // create a stage to put the file in - statement.execute("CREATE OR REPLACE STAGE largefile_stage"); - assertTrue( - "Failed to put a file", - statement.execute("PUT file://" + sourceFilePath + " @largefile_stage")); - - // check that file exists in stage after PUT - findFile(statement, "ls @largefile_stage/"); - - // create a new table with columns matching CSV file - statement.execute("create or replace table large_table (colA string)"); - // copy rows from file into table - statement.execute("copy into large_table from @largefile_stage/largeFile.csv.gz"); - // copy back from table into different stage - statement.execute("create or replace stage extra_stage"); - statement.execute("copy into @extra_stage/bigFile.csv.gz from large_table single=true"); - - // get file from new stage - assertTrue( - "Failed to get files", - statement.execute( - "GET @extra_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); - - // Make sure that the downloaded file exists; it should be gzip compressed - File downloaded = new File(destFolderCanonicalPathWithSeparator + "bigFile.csv.gz"); - assertTrue(downloaded.exists()); - - // unzip the file - Process p = - Runtime.getRuntime() - .exec("gzip -d " + destFolderCanonicalPathWithSeparator + "bigFile.csv.gz"); - p.waitFor(); - - // compare the original file with the file that's been uploaded, copied into a table, copied - // back into a stage, - // downloaded, and unzipped - File unzipped = new File(destFolderCanonicalPathWithSeparator + "bigFile.csv"); - assertEquals(largeTempFile.length(), unzipped.length()); - assertTrue(FileUtils.contentEquals(largeTempFile, unzipped)); - } finally { - statement.execute("DROP STAGE IF EXISTS largefile_stage"); - statement.execute("DROP STAGE IF EXISTS extra_stage"); - statement.execute("DROP TABLE IF EXISTS large_table"); - } - } - } - - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testPutOverwrite() throws Throwable { - // create 2 files: an original, and one that will overwrite the original - File file1 = tmpFolder.newFile("testfile.csv"); - try (BufferedWriter bw = new BufferedWriter(new FileWriter(file1))) { - bw.write("Writing original file content. This should get overwritten."); - } - - File file2 = tmpFolder2.newFile("testfile.csv"); - try (BufferedWriter bw = new BufferedWriter(new FileWriter(file2))) { - bw.write("This is all new! This should be the result of the overwriting."); - } - - String sourceFilePathOriginal = file1.getCanonicalPath(); - String sourceFilePathOverwrite = file2.getCanonicalPath(); - - File destFolder = tmpFolder.newFolder(); - String destFolderCanonicalPath = destFolder.getCanonicalPath(); - String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; - - List accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount"); - for (int i = 0; i < accounts.size(); i++) { - try (Connection connection = getConnection(accounts.get(i)); - Statement statement = connection.createStatement()) { - try { - statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); - - // create a stage to put the file in - statement.execute("CREATE OR REPLACE STAGE testing_stage"); - assertTrue( - "Failed to put a file", - statement.execute("PUT file://" + sourceFilePathOriginal + " @testing_stage")); - // check that file exists in stage after PUT - findFile(statement, "ls @testing_stage/"); - - // put another file in same stage with same filename with overwrite = true - assertTrue( - "Failed to put a file", - statement.execute( - "PUT file://" + sourceFilePathOverwrite + " @testing_stage overwrite=true")); - - // check that file exists in stage after PUT - findFile(statement, "ls @testing_stage/"); - - // get file from new stage - assertTrue( - "Failed to get files", - statement.execute( - "GET @testing_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); - - // Make sure that the downloaded file exists; it should be gzip compressed - File downloaded = new File(destFolderCanonicalPathWithSeparator + "testfile.csv.gz"); - assertTrue(downloaded.exists()); - - // unzip the file - Process p = - Runtime.getRuntime() - .exec("gzip -d " + destFolderCanonicalPathWithSeparator + "testfile.csv.gz"); - p.waitFor(); - - File unzipped = new File(destFolderCanonicalPathWithSeparator + "testfile.csv"); - assertTrue(FileUtils.contentEqualsIgnoreEOL(file2, unzipped, null)); - } finally { - statement.execute("DROP TABLE IF EXISTS testLoadToLocalFS"); - } - } - } - } - - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testPut() throws Throwable { - - List accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount"); - for (int i = 0; i < accounts.size(); i++) { - try (Connection connection = getConnection(accounts.get(i)); - Statement statement = connection.createStatement()) { - try { - // load file test - // create a unique data file name by using current timestamp in millis - statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); - // test external table load - statement.execute("CREATE OR REPLACE TABLE testLoadToLocalFS(a number)"); - - // put files - assertTrue( - "Failed to put a file", - statement.execute( - "PUT file://" - + getFullPathFileInResource(TEST_DATA_FILE) - + " @%testLoadToLocalFS/orders parallel=10")); - - try (ResultSet resultSet = statement.getResultSet()) { - - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - - // assert column count - assertTrue(resultSetMetaData.getColumnCount() > 0); - - assertTrue(resultSet.next()); // one row - assertFalse(resultSet.next()); - } - findFile( - statement, "ls @%testLoadToLocalFS/ pattern='.*orders/" + TEST_DATA_FILE + ".g.*'"); - - // remove files - try (ResultSet resultSet = - statement.executeQuery( - "rm @%testLoadToLocalFS/ pattern='.*orders/" + TEST_DATA_FILE + ".g.*'")) { - - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - - // assert column count - assertTrue(resultSetMetaData.getColumnCount() >= 1); - - // assert we get 1 row for the file we copied - assertTrue(resultSet.next()); - assertNotNull(resultSet.getString(1)); - assertFalse(resultSet.next()); - try { - resultSet.getString(1); // no more row - fail("must fail"); - } catch (SQLException ex) { - assertEquals( - (int) ErrorCode.COLUMN_DOES_NOT_EXIST.getMessageCode(), ex.getErrorCode()); - } - - Thread.sleep(100); - } - // show files again - try (ResultSet resultSet = - statement.executeQuery("ls @%testLoadToLocalFS/ pattern='.*orders/orders.*'")) { - - // assert we get 0 row - assertFalse(resultSet.next()); - } - } finally { - statement.execute("DROP TABLE IF EXISTS testLoadToLocalFS"); - } - } - } - } - static void findFile(Statement statement, String checkSQL) throws Throwable { boolean fileFound = false; @@ -2664,55 +2388,6 @@ public void testSnow31104() throws Throwable { } } - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testPutGet() throws Throwable { - - List accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount"); - for (int i = 0; i < accounts.size(); i++) { - try (Connection connection = getConnection(accounts.get(i)); - Statement statement = connection.createStatement()) { - try { - String sourceFilePath = getFullPathFileInResource(TEST_DATA_FILE); - - File destFolder = tmpFolder.newFolder(); - String destFolderCanonicalPath = destFolder.getCanonicalPath(); - String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; - - statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); - statement.execute("CREATE OR REPLACE STAGE testPutGet_stage"); - - assertTrue( - "Failed to put a file", - statement.execute("PUT file://" + sourceFilePath + " @testPutGet_stage")); - - findFile(statement, "ls @testPutGet_stage/"); - - // download the file we just uploaded to stage - assertTrue( - "Failed to get a file", - statement.execute( - "GET @testPutGet_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); - - // Make sure that the downloaded file exists, it should be gzip compressed - File downloaded = new File(destFolderCanonicalPathWithSeparator + TEST_DATA_FILE + ".gz"); - assertTrue(downloaded.exists()); - - Process p = - Runtime.getRuntime() - .exec("gzip -d " + destFolderCanonicalPathWithSeparator + TEST_DATA_FILE + ".gz"); - p.waitFor(); - - File original = new File(sourceFilePath); - File unzipped = new File(destFolderCanonicalPathWithSeparator + TEST_DATA_FILE); - assertEquals(original.length(), unzipped.length()); - } finally { - statement.execute("DROP STAGE IF EXISTS testGetPut_stage"); - } - } - } - } - /** * Tests unencrypted named stages, which don't use client-side encryption to enable data lake * scenarios. Unencrypted named stages are specified with encryption=(TYPE='SNOWFLAKE_SSE'). diff --git a/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverLatestIT.java b/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverLatestIT.java index 05df191d5..f6d1057f1 100644 --- a/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/SnowflakeDriverLatestIT.java @@ -9,6 +9,7 @@ import static net.snowflake.client.jdbc.SnowflakeResultSetSerializableV1.mapper; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1774,4 +1775,330 @@ public void shouldLoadDriverWithDisabledTelemetryOob() throws ClassNotFoundExcep assertFalse(TelemetryService.getInstance().isEnabled()); assertFalse(TelemetryService.getInstance().isHTAPEnabled()); } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testPutWithWildcardGCP() throws Throwable { + Properties _connectionProperties = new Properties(); + _connectionProperties.put("inject_wait_in_put", 5); + _connectionProperties.put("ssl", "off"); + try (Connection connection = + getConnection( + DONT_INJECT_SOCKET_TIMEOUT, _connectionProperties, false, false, "gcpaccount"); + Statement statement = connection.createStatement()) { + try { + String sourceFilePath = getFullPathFileInResource(TEST_DATA_FILE); + // replace file name with wildcard character + sourceFilePath = sourceFilePath.replace("orders_100.csv", "orders_10*.csv"); + + File destFolder = tmpFolder.newFolder(); + String destFolderCanonicalPath = destFolder.getCanonicalPath(); + String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; + statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); + statement.execute("CREATE OR REPLACE STAGE wildcard_stage"); + assertTrue( + "Failed to put a file", + statement.execute("PUT file://" + sourceFilePath + " @wildcard_stage")); + + findFile(statement, "ls @wildcard_stage/"); + + assertTrue( + "Failed to get files", + statement.execute( + "GET @wildcard_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); + + File downloaded; + // download the files we just uploaded to stage + for (int i = 0; i < fileNames.length; i++) { + // Make sure that the downloaded file exists, it should be gzip compressed + downloaded = new File(destFolderCanonicalPathWithSeparator + fileNames[i] + ".gz"); + assertTrue(downloaded.exists()); + + Process p = + Runtime.getRuntime() + .exec("gzip -d " + destFolderCanonicalPathWithSeparator + fileNames[i] + ".gz"); + p.waitFor(); + + String individualFilePath = sourceFilePath.replace("orders_10*.csv", fileNames[i]); + + File original = new File(individualFilePath); + File unzipped = new File(destFolderCanonicalPathWithSeparator + fileNames[i]); + assertEquals(original.length(), unzipped.length()); + assertTrue(FileUtils.contentEquals(original, unzipped)); + } + } finally { + statement.execute("DROP STAGE IF EXISTS wildcard_stage"); + } + } + } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testPutGetLargeFileGCP() throws Throwable { + try (Connection connection = getConnection("gcpaccount"); + Statement statement = connection.createStatement()) { + try { + File destFolder = tmpFolder.newFolder(); + String destFolderCanonicalPath = destFolder.getCanonicalPath(); + String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; + + File largeTempFile = tmpFolder.newFile("largeFile.csv"); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(largeTempFile))) { + bw.write("Creating large test file for GCP PUT/GET test"); + bw.write(System.lineSeparator()); + bw.write("Creating large test file for GCP PUT/GET test"); + bw.write(System.lineSeparator()); + } + File largeTempFile2 = tmpFolder.newFile("largeFile2.csv"); + + String sourceFilePath = largeTempFile.getCanonicalPath(); + + // copy info from 1 file to another and continue doubling file size until we reach ~1.5GB, + // which is a large file + for (int i = 0; i < 12; i++) { + copyContentFrom(largeTempFile, largeTempFile2); + copyContentFrom(largeTempFile2, largeTempFile); + } + + statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); + + // create a stage to put the file in + statement.execute("CREATE OR REPLACE STAGE largefile_stage"); + assertTrue( + "Failed to put a file", + statement.execute("PUT file://" + sourceFilePath + " @largefile_stage")); + + // check that file exists in stage after PUT + findFile(statement, "ls @largefile_stage/"); + + // create a new table with columns matching CSV file + statement.execute("create or replace table large_table (colA string)"); + // copy rows from file into table + statement.execute("copy into large_table from @largefile_stage/largeFile.csv.gz"); + // copy back from table into different stage + statement.execute("create or replace stage extra_stage"); + statement.execute("copy into @extra_stage/bigFile.csv.gz from large_table single=true"); + + // get file from new stage + assertTrue( + "Failed to get files", + statement.execute( + "GET @extra_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); + + // Make sure that the downloaded file exists; it should be gzip compressed + File downloaded = new File(destFolderCanonicalPathWithSeparator + "bigFile.csv.gz"); + assertTrue(downloaded.exists()); + + // unzip the file + Process p = + Runtime.getRuntime() + .exec("gzip -d " + destFolderCanonicalPathWithSeparator + "bigFile.csv.gz"); + p.waitFor(); + + // compare the original file with the file that's been uploaded, copied into a table, copied + // back into a stage, + // downloaded, and unzipped + File unzipped = new File(destFolderCanonicalPathWithSeparator + "bigFile.csv"); + assertEquals(largeTempFile.length(), unzipped.length()); + assertTrue(FileUtils.contentEquals(largeTempFile, unzipped)); + } finally { + statement.execute("DROP STAGE IF EXISTS largefile_stage"); + statement.execute("DROP STAGE IF EXISTS extra_stage"); + statement.execute("DROP TABLE IF EXISTS large_table"); + } + } + } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testPutOverwrite() throws Throwable { + // create 2 files: an original, and one that will overwrite the original + File file1 = tmpFolder.newFile("testfile.csv"); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(file1))) { + bw.write("Writing original file content. This should get overwritten."); + } + + File file2 = tmpFolder2.newFile("testfile.csv"); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(file2))) { + bw.write("This is all new! This should be the result of the overwriting."); + } + + String sourceFilePathOriginal = file1.getCanonicalPath(); + String sourceFilePathOverwrite = file2.getCanonicalPath(); + + File destFolder = tmpFolder.newFolder(); + String destFolderCanonicalPath = destFolder.getCanonicalPath(); + String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; + + List accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount"); + for (int i = 0; i < accounts.size(); i++) { + try (Connection connection = getConnection(accounts.get(i)); + Statement statement = connection.createStatement()) { + try { + statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); + + // create a stage to put the file in + statement.execute("CREATE OR REPLACE STAGE testing_stage"); + assertTrue( + "Failed to put a file", + statement.execute("PUT file://" + sourceFilePathOriginal + " @testing_stage")); + // check that file exists in stage after PUT + findFile(statement, "ls @testing_stage/"); + + // put another file in same stage with same filename with overwrite = true + assertTrue( + "Failed to put a file", + statement.execute( + "PUT file://" + sourceFilePathOverwrite + " @testing_stage overwrite=true")); + + // check that file exists in stage after PUT + findFile(statement, "ls @testing_stage/"); + + // get file from new stage + assertTrue( + "Failed to get files", + statement.execute( + "GET @testing_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); + + // Make sure that the downloaded file exists; it should be gzip compressed + File downloaded = new File(destFolderCanonicalPathWithSeparator + "testfile.csv.gz"); + assertTrue(downloaded.exists()); + + // unzip the file + Process p = + Runtime.getRuntime() + .exec("gzip -d " + destFolderCanonicalPathWithSeparator + "testfile.csv.gz"); + p.waitFor(); + + File unzipped = new File(destFolderCanonicalPathWithSeparator + "testfile.csv"); + assertTrue(FileUtils.contentEqualsIgnoreEOL(file2, unzipped, null)); + } finally { + statement.execute("DROP TABLE IF EXISTS testLoadToLocalFS"); + } + } + } + } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testPut() throws Throwable { + + List accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount"); + for (int i = 0; i < accounts.size(); i++) { + try (Connection connection = getConnection(accounts.get(i)); + Statement statement = connection.createStatement()) { + try { + // load file test + // create a unique data file name by using current timestamp in millis + statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); + // test external table load + statement.execute("CREATE OR REPLACE TABLE testLoadToLocalFS(a number)"); + + // put files + assertTrue( + "Failed to put a file", + statement.execute( + "PUT file://" + + getFullPathFileInResource(TEST_DATA_FILE) + + " @%testLoadToLocalFS/orders parallel=10")); + + try (ResultSet resultSet = statement.getResultSet()) { + + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + + // assert column count + assertTrue(resultSetMetaData.getColumnCount() > 0); + + assertTrue(resultSet.next()); // one row + assertFalse(resultSet.next()); + } + findFile( + statement, "ls @%testLoadToLocalFS/ pattern='.*orders/" + TEST_DATA_FILE + ".g.*'"); + + // remove files + try (ResultSet resultSet = + statement.executeQuery( + "rm @%testLoadToLocalFS/ pattern='.*orders/" + TEST_DATA_FILE + ".g.*'")) { + + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + + // assert column count + assertTrue(resultSetMetaData.getColumnCount() >= 1); + + // assert we get 1 row for the file we copied + assertTrue(resultSet.next()); + assertNotNull(resultSet.getString(1)); + assertFalse(resultSet.next()); + try { + resultSet.getString(1); // no more row + fail("must fail"); + } catch (SQLException ex) { + assertEquals( + (int) ErrorCode.COLUMN_DOES_NOT_EXIST.getMessageCode(), ex.getErrorCode()); + } + + Thread.sleep(100); + } + // show files again + try (ResultSet resultSet = + statement.executeQuery("ls @%testLoadToLocalFS/ pattern='.*orders/orders.*'")) { + + // assert we get 0 row + assertFalse(resultSet.next()); + } + } finally { + statement.execute("DROP TABLE IF EXISTS testLoadToLocalFS"); + } + } + } + } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testPutGet() throws Throwable { + + List accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount"); + for (int i = 0; i < accounts.size(); i++) { + try (Connection connection = getConnection(accounts.get(i)); + Statement statement = connection.createStatement()) { + try { + String sourceFilePath = getFullPathFileInResource(TEST_DATA_FILE); + + File destFolder = tmpFolder.newFolder(); + String destFolderCanonicalPath = destFolder.getCanonicalPath(); + String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator; + + statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false"); + statement.execute("CREATE OR REPLACE STAGE testPutGet_stage"); + + assertTrue( + "Failed to put a file", + statement.execute("PUT file://" + sourceFilePath + " @testPutGet_stage")); + + findFile(statement, "ls @testPutGet_stage/"); + + // download the file we just uploaded to stage + assertTrue( + "Failed to get a file", + statement.execute( + "GET @testPutGet_stage 'file://" + destFolderCanonicalPath + "' parallel=8")); + + // Make sure that the downloaded file exists, it should be gzip compressed + File downloaded = new File(destFolderCanonicalPathWithSeparator + TEST_DATA_FILE + ".gz"); + assertTrue(downloaded.exists()); + + Process p = + Runtime.getRuntime() + .exec("gzip -d " + destFolderCanonicalPathWithSeparator + TEST_DATA_FILE + ".gz"); + p.waitFor(); + + File original = new File(sourceFilePath); + File unzipped = new File(destFolderCanonicalPathWithSeparator + TEST_DATA_FILE); + assertEquals(original.length(), unzipped.length()); + } finally { + statement.execute("DROP STAGE IF EXISTS testGetPut_stage"); + } + } + } + } }