Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SNOW-1561613: Gather more data from failing Jenkins tests #1852

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
+335 −330
Draft

SNOW-1561613: Gather more data from failing Jenkins tests #1852

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
adfe755
SNOW-1561613: Gather more data from failing Jenkins tests
sfc-gh-dprzybysz Jul 31, 2024
43ac8f0
SNOW-1561613: Log GCS digest
sfc-gh-dprzybysz Jul 31, 2024
7d8e84c
SNOW-1561613: Move failing OldDriver tests with GCP to LatestIT
sfc-gh-dprzybysz Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -993,12 +993,13 @@ private void uploadWithPresignedUrl(
SqlState.INTERNAL_ERROR,
"Unexpected: upload presigned URL invalid");
} catch (Exception e) {
e.printStackTrace();
throw new SnowflakeSQLLoggedException(
queryId,
session,
ErrorCode.INTERNAL_ERROR.getMessageCode(),
SqlState.INTERNAL_ERROR,
"Unexpected: upload with presigned url failed");
"Unexpected: upload with presigned url failed: " + e.getMessage());
}
}

Expand Down Expand Up @@ -1279,6 +1280,7 @@ private AbstractMap.SimpleEntry<String, String> parseEncryptionData(
@Override
public void addDigestMetadata(StorageObjectMetadata meta, String digest) {
if (!SnowflakeUtil.isBlank(digest)) {
System.out.println("Setting digest to '" + digest + "'");
meta.addUserMetadata("sfc-digest", digest);
}
}
Expand Down
334 changes: 5 additions & 329 deletions src/test/java/net/snowflake/client/jdbc/SnowflakeDriverIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -733,61 +733,6 @@ public void testDBMetadata() throws Throwable {
}
}

@Test
@ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class)
public void testPutWithWildcardGCP() throws Throwable {
Properties _connectionProperties = new Properties();
_connectionProperties.put("inject_wait_in_put", 5);
_connectionProperties.put("ssl", "off");
try (Connection connection =
getConnection(
DONT_INJECT_SOCKET_TIMEOUT, _connectionProperties, false, false, "gcpaccount");
Statement statement = connection.createStatement()) {
try {
String sourceFilePath = getFullPathFileInResource(TEST_DATA_FILE);
// replace file name with wildcard character
sourceFilePath = sourceFilePath.replace("orders_100.csv", "orders_10*.csv");

File destFolder = tmpFolder.newFolder();
String destFolderCanonicalPath = destFolder.getCanonicalPath();
String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator;
statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false");
statement.execute("CREATE OR REPLACE STAGE wildcard_stage");
assertTrue(
"Failed to put a file",
statement.execute("PUT file://" + sourceFilePath + " @wildcard_stage"));

findFile(statement, "ls @wildcard_stage/");

assertTrue(
"Failed to get files",
statement.execute(
"GET @wildcard_stage 'file://" + destFolderCanonicalPath + "' parallel=8"));

File downloaded;
// download the files we just uploaded to stage
for (int i = 0; i < fileNames.length; i++) {
// Make sure that the downloaded file exists, it should be gzip compressed
downloaded = new File(destFolderCanonicalPathWithSeparator + fileNames[i] + ".gz");
assertTrue(downloaded.exists());

Process p =
Runtime.getRuntime()
.exec("gzip -d " + destFolderCanonicalPathWithSeparator + fileNames[i] + ".gz");
p.waitFor();

String individualFilePath = sourceFilePath.replace("orders_10*.csv", fileNames[i]);

File original = new File(individualFilePath);
File unzipped = new File(destFolderCanonicalPathWithSeparator + fileNames[i]);
assertEquals(original.length(), unzipped.length());
assertTrue(FileUtils.contentEquals(original, unzipped));
}
} finally {
statement.execute("DROP STAGE IF EXISTS wildcard_stage");
}
}
}

/**
* helper function for creating large file in Java. Copies info from 1 file to another
Expand All @@ -807,227 +752,6 @@ private void copyContentFrom(File file1, File file2) throws Exception {
}
}

@Test
@ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class)
public void testPutGetLargeFileGCP() throws Throwable {
try (Connection connection = getConnection("gcpaccount");
Statement statement = connection.createStatement()) {
try {
File destFolder = tmpFolder.newFolder();
String destFolderCanonicalPath = destFolder.getCanonicalPath();
String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator;

File largeTempFile = tmpFolder.newFile("largeFile.csv");
try (BufferedWriter bw = new BufferedWriter(new FileWriter(largeTempFile))) {
bw.write("Creating large test file for GCP PUT/GET test");
bw.write(System.lineSeparator());
bw.write("Creating large test file for GCP PUT/GET test");
bw.write(System.lineSeparator());
}
File largeTempFile2 = tmpFolder.newFile("largeFile2.csv");

String sourceFilePath = largeTempFile.getCanonicalPath();

// copy info from 1 file to another and continue doubling file size until we reach ~1.5GB,
// which is a large file
for (int i = 0; i < 12; i++) {
copyContentFrom(largeTempFile, largeTempFile2);
copyContentFrom(largeTempFile2, largeTempFile);
}

statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false");

// create a stage to put the file in
statement.execute("CREATE OR REPLACE STAGE largefile_stage");
assertTrue(
"Failed to put a file",
statement.execute("PUT file://" + sourceFilePath + " @largefile_stage"));

// check that file exists in stage after PUT
findFile(statement, "ls @largefile_stage/");

// create a new table with columns matching CSV file
statement.execute("create or replace table large_table (colA string)");
// copy rows from file into table
statement.execute("copy into large_table from @largefile_stage/largeFile.csv.gz");
// copy back from table into different stage
statement.execute("create or replace stage extra_stage");
statement.execute("copy into @extra_stage/bigFile.csv.gz from large_table single=true");

// get file from new stage
assertTrue(
"Failed to get files",
statement.execute(
"GET @extra_stage 'file://" + destFolderCanonicalPath + "' parallel=8"));

// Make sure that the downloaded file exists; it should be gzip compressed
File downloaded = new File(destFolderCanonicalPathWithSeparator + "bigFile.csv.gz");
assertTrue(downloaded.exists());

// unzip the file
Process p =
Runtime.getRuntime()
.exec("gzip -d " + destFolderCanonicalPathWithSeparator + "bigFile.csv.gz");
p.waitFor();

// compare the original file with the file that's been uploaded, copied into a table, copied
// back into a stage,
// downloaded, and unzipped
File unzipped = new File(destFolderCanonicalPathWithSeparator + "bigFile.csv");
assertEquals(largeTempFile.length(), unzipped.length());
assertTrue(FileUtils.contentEquals(largeTempFile, unzipped));
} finally {
statement.execute("DROP STAGE IF EXISTS largefile_stage");
statement.execute("DROP STAGE IF EXISTS extra_stage");
statement.execute("DROP TABLE IF EXISTS large_table");
}
}
}

@Test
@ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class)
public void testPutOverwrite() throws Throwable {
// create 2 files: an original, and one that will overwrite the original
File file1 = tmpFolder.newFile("testfile.csv");
try (BufferedWriter bw = new BufferedWriter(new FileWriter(file1))) {
bw.write("Writing original file content. This should get overwritten.");
}

File file2 = tmpFolder2.newFile("testfile.csv");
try (BufferedWriter bw = new BufferedWriter(new FileWriter(file2))) {
bw.write("This is all new! This should be the result of the overwriting.");
}

String sourceFilePathOriginal = file1.getCanonicalPath();
String sourceFilePathOverwrite = file2.getCanonicalPath();

File destFolder = tmpFolder.newFolder();
String destFolderCanonicalPath = destFolder.getCanonicalPath();
String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator;

List<String> accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount");
for (int i = 0; i < accounts.size(); i++) {
try (Connection connection = getConnection(accounts.get(i));
Statement statement = connection.createStatement()) {
try {
statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false");

// create a stage to put the file in
statement.execute("CREATE OR REPLACE STAGE testing_stage");
assertTrue(
"Failed to put a file",
statement.execute("PUT file://" + sourceFilePathOriginal + " @testing_stage"));
// check that file exists in stage after PUT
findFile(statement, "ls @testing_stage/");

// put another file in same stage with same filename with overwrite = true
assertTrue(
"Failed to put a file",
statement.execute(
"PUT file://" + sourceFilePathOverwrite + " @testing_stage overwrite=true"));

// check that file exists in stage after PUT
findFile(statement, "ls @testing_stage/");

// get file from new stage
assertTrue(
"Failed to get files",
statement.execute(
"GET @testing_stage 'file://" + destFolderCanonicalPath + "' parallel=8"));

// Make sure that the downloaded file exists; it should be gzip compressed
File downloaded = new File(destFolderCanonicalPathWithSeparator + "testfile.csv.gz");
assertTrue(downloaded.exists());

// unzip the file
Process p =
Runtime.getRuntime()
.exec("gzip -d " + destFolderCanonicalPathWithSeparator + "testfile.csv.gz");
p.waitFor();

File unzipped = new File(destFolderCanonicalPathWithSeparator + "testfile.csv");
assertTrue(FileUtils.contentEqualsIgnoreEOL(file2, unzipped, null));
} finally {
statement.execute("DROP TABLE IF EXISTS testLoadToLocalFS");
}
}
}
}

@Test
@ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class)
public void testPut() throws Throwable {

List<String> accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount");
for (int i = 0; i < accounts.size(); i++) {
try (Connection connection = getConnection(accounts.get(i));
Statement statement = connection.createStatement()) {
try {
// load file test
// create a unique data file name by using current timestamp in millis
statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false");
// test external table load
statement.execute("CREATE OR REPLACE TABLE testLoadToLocalFS(a number)");

// put files
assertTrue(
"Failed to put a file",
statement.execute(
"PUT file://"
+ getFullPathFileInResource(TEST_DATA_FILE)
+ " @%testLoadToLocalFS/orders parallel=10"));

try (ResultSet resultSet = statement.getResultSet()) {

ResultSetMetaData resultSetMetaData = resultSet.getMetaData();

// assert column count
assertTrue(resultSetMetaData.getColumnCount() > 0);

assertTrue(resultSet.next()); // one row
assertFalse(resultSet.next());
}
findFile(
statement, "ls @%testLoadToLocalFS/ pattern='.*orders/" + TEST_DATA_FILE + ".g.*'");

// remove files
try (ResultSet resultSet =
statement.executeQuery(
"rm @%testLoadToLocalFS/ pattern='.*orders/" + TEST_DATA_FILE + ".g.*'")) {

ResultSetMetaData resultSetMetaData = resultSet.getMetaData();

// assert column count
assertTrue(resultSetMetaData.getColumnCount() >= 1);

// assert we get 1 row for the file we copied
assertTrue(resultSet.next());
assertNotNull(resultSet.getString(1));
assertFalse(resultSet.next());
try {
resultSet.getString(1); // no more row
fail("must fail");
} catch (SQLException ex) {
assertEquals(
(int) ErrorCode.COLUMN_DOES_NOT_EXIST.getMessageCode(), ex.getErrorCode());
}

Thread.sleep(100);
}
// show files again
try (ResultSet resultSet =
statement.executeQuery("ls @%testLoadToLocalFS/ pattern='.*orders/orders.*'")) {

// assert we get 0 row
assertFalse(resultSet.next());
}
} finally {
statement.execute("DROP TABLE IF EXISTS testLoadToLocalFS");
}
}
}
}

static void findFile(Statement statement, String checkSQL) throws Throwable {
boolean fileFound = false;

Expand All @@ -1038,16 +762,17 @@ static void findFile(Statement statement, String checkSQL) throws Throwable {

if (resultSet.next()) {
fileFound = true;

// assert the first column not null
assertNotNull("Null result", resultSet.getString(1));

break;
}
// give enough time for s3 eventual consistency for US region
Thread.sleep(1000);
assertTrue("Could not find a file", fileFound);

// assert the first column not null
assertNotNull("Null result", resultSet.getString(1));
}
}
assertTrue("Could not find a file", fileFound);
}

@Test
Expand Down Expand Up @@ -2663,55 +2388,6 @@ public void testSnow31104() throws Throwable {
}
}

@Test
@ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class)
public void testPutGet() throws Throwable {

List<String> accounts = Arrays.asList(null, "s3testaccount", "azureaccount", "gcpaccount");
for (int i = 0; i < accounts.size(); i++) {
try (Connection connection = getConnection(accounts.get(i));
Statement statement = connection.createStatement()) {
try {
String sourceFilePath = getFullPathFileInResource(TEST_DATA_FILE);

File destFolder = tmpFolder.newFolder();
String destFolderCanonicalPath = destFolder.getCanonicalPath();
String destFolderCanonicalPathWithSeparator = destFolderCanonicalPath + File.separator;

statement.execute("alter session set ENABLE_GCP_PUT_EXCEPTION_FOR_OLD_DRIVERS=false");
statement.execute("CREATE OR REPLACE STAGE testPutGet_stage");

assertTrue(
"Failed to put a file",
statement.execute("PUT file://" + sourceFilePath + " @testPutGet_stage"));

findFile(statement, "ls @testPutGet_stage/");

// download the file we just uploaded to stage
assertTrue(
"Failed to get a file",
statement.execute(
"GET @testPutGet_stage 'file://" + destFolderCanonicalPath + "' parallel=8"));

// Make sure that the downloaded file exists, it should be gzip compressed
File downloaded = new File(destFolderCanonicalPathWithSeparator + TEST_DATA_FILE + ".gz");
assertTrue(downloaded.exists());

Process p =
Runtime.getRuntime()
.exec("gzip -d " + destFolderCanonicalPathWithSeparator + TEST_DATA_FILE + ".gz");
p.waitFor();

File original = new File(sourceFilePath);
File unzipped = new File(destFolderCanonicalPathWithSeparator + TEST_DATA_FILE);
assertEquals(original.length(), unzipped.length());
} finally {
statement.execute("DROP STAGE IF EXISTS testGetPut_stage");
}
}
}
}

/**
* Tests unencrypted named stages, which don't use client-side encryption to enable data lake
* scenarios. Unencrypted named stages are specified with encryption=(TYPE='SNOWFLAKE_SSE').
Expand Down
Loading
Loading