From 6477407d96f84f6874de18ea0653896f7ab6aae1 Mon Sep 17 00:00:00 2001 From: Piotr Fus Date: Mon, 22 Jul 2024 12:55:56 +0200 Subject: [PATCH] SNOW-1431870 Add cloud integration for GCM encryption --- .../jdbc/SnowflakeFileTransferAgent.java | 4 +- .../jdbc/cloud/storage/DecryptionHelper.java | 137 ++++++++++++++ .../cloud/storage/SnowflakeAzureClient.java | 126 ++++--------- .../cloud/storage/SnowflakeGCSClient.java | 146 +++++---------- .../jdbc/cloud/storage/SnowflakeS3Client.java | 116 ++++++++---- .../cloud/storage/SnowflakeStorageClient.java | 6 +- .../client/jdbc/cloud/storage/StageInfo.java | 73 +++++++- .../cloud/storage/StorageClientFactory.java | 9 +- .../cloud/storage/StorageClientHelper.java | 177 ++++++++++++++++++ .../snowflake/client/AbstractDriverIT.java | 4 + .../storage/CloudStorageClientLatestIT.java | 3 + 11 files changed, 561 insertions(+), 240 deletions(-) create mode 100644 src/main/java/net/snowflake/client/jdbc/cloud/storage/DecryptionHelper.java create mode 100644 src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientHelper.java diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 751b47d19..2a0e9df15 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1106,6 +1106,7 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf isClientSideEncrypted = jsonNode.path("data").path("stageInfo").path("isClientSideEncrypted").asBoolean(true); } + String ciphers = jsonNode.path("data").path("stageInfo").path("ciphers").asText(); // endPoint is currently known to be set for Azure stages or S3. For S3 it will be set // specifically @@ -1166,7 +1167,8 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf stageRegion, endPoint, stgAcct, - isClientSideEncrypted); + isClientSideEncrypted, + ciphers); // Setup pre-signed URL into stage info if pre-signed URL is returned. if (stageInfo.getStageType() == StageInfo.StageType.GCS) { diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/DecryptionHelper.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/DecryptionHelper.java new file mode 100644 index 000000000..1a8c367cd --- /dev/null +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/DecryptionHelper.java @@ -0,0 +1,137 @@ +package net.snowflake.client.jdbc.cloud.storage; + +import com.google.common.base.Strings; +import java.io.File; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import net.snowflake.client.core.SFBaseSession; +import net.snowflake.client.jdbc.ErrorCode; +import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; +import net.snowflake.client.log.SFLogger; +import net.snowflake.client.log.SFLoggerFactory; +import net.snowflake.common.core.RemoteStoreFileEncryptionMaterial; +import net.snowflake.common.core.SqlState; + +class DecryptionHelper { + private static final SFLogger logger = SFLoggerFactory.getLogger(SnowflakeGCSClient.class); + + private final String queryId; + private final SFBaseSession session; + private final String key; + private final String keyIv; + private final String dataIv; + private final String keyAad; + private final String dataAad; + private final StageInfo.Ciphers ciphers; + + private DecryptionHelper( + String queryId, + SFBaseSession session, + String key, + String keyIv, + String dataIv, + String keyAad, + String dataAad, + StageInfo.Ciphers ciphers) { + this.queryId = queryId; + this.session = session; + this.key = key; + this.keyIv = keyIv; + this.dataIv = dataIv; + this.keyAad = keyAad; + this.dataAad = dataAad; + this.ciphers = ciphers; + } + + static DecryptionHelper forCbc( + String queryId, SFBaseSession session, String key, String contentIv) + throws SnowflakeSQLLoggedException { + if (Strings.isNullOrEmpty(key) || Strings.isNullOrEmpty(contentIv)) { + throw exception(queryId, session); + } + return new DecryptionHelper( + queryId, session, key, null, contentIv, null, null, StageInfo.Ciphers.AESECB_AESCBC); + } + + static DecryptionHelper forGcm( + String queryId, + SFBaseSession session, + String key, + String keyIv, + String dataIv, + String keyAad, + String dataAad) + throws SnowflakeSQLLoggedException { + if (Strings.isNullOrEmpty(key) + || Strings.isNullOrEmpty(keyIv) + || Strings.isNullOrEmpty(dataIv) + || keyAad == null + || dataAad == null) { + throw exception(queryId, session); + } + return new DecryptionHelper( + queryId, session, key, keyIv, dataIv, keyAad, dataAad, StageInfo.Ciphers.AESGCM_AESGCM); + } + + void validate() throws SnowflakeSQLLoggedException { + if (key == null + || dataIv == null + || (ciphers == StageInfo.Ciphers.AESGCM_AESGCM && keyIv == null)) { + throw new SnowflakeSQLLoggedException( + queryId, + session, + ErrorCode.INTERNAL_ERROR.getMessageCode(), + SqlState.INTERNAL_ERROR, + "File metadata incomplete"); + } + } + + void decryptFile(File file, RemoteStoreFileEncryptionMaterial encMat) + throws SnowflakeSQLLoggedException { + try { + switch (ciphers) { + case AESECB_AESCBC: + EncryptionProvider.decrypt(file, key, dataIv, encMat); + case AESGCM_AESGCM: + GcmEncryptionProvider.decryptFile(file, key, dataIv, keyIv, encMat, dataAad, keyAad); + default: + throw new IllegalArgumentException("unsuported ciphers: " + ciphers); + } + } catch (Exception ex) { + logger.error("Error decrypting file", ex); + throw new SnowflakeSQLLoggedException( + queryId, + session, + ErrorCode.INTERNAL_ERROR.getMessageCode(), + SqlState.INTERNAL_ERROR, + "Cannot decrypt file"); + } + } + + InputStream decryptStream(InputStream inputStream, RemoteStoreFileEncryptionMaterial encMat) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidKeyException, + BadPaddingException, IllegalBlockSizeException, InvalidAlgorithmParameterException { + switch (ciphers) { + case AESGCM_AESGCM: + return GcmEncryptionProvider.decryptStream( + inputStream, key, dataIv, keyIv, encMat, dataAad, keyAad); + case AESECB_AESCBC: + return EncryptionProvider.decryptStream(inputStream, key, dataIv, encMat); + } + throw new IllegalArgumentException("unsupported ciphers: " + ciphers); + } + + private static SnowflakeSQLLoggedException exception(String queryId, SFBaseSession session) { + return new SnowflakeSQLLoggedException( + queryId, + session, + ErrorCode.INTERNAL_ERROR.getMessageCode(), + SqlState.INTERNAL_ERROR, + "File metadata incomplete"); + } +} diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeAzureClient.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeAzureClient.java index cdf303bbd..da0a457b4 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeAzureClient.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeAzureClient.java @@ -6,10 +6,6 @@ import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED; import static net.snowflake.client.jdbc.SnowflakeUtil.systemGetProperty; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.azure.storage.OperationContext; import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentialsAnonymous; @@ -33,8 +29,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.InvalidKeyException; -import java.util.AbstractMap; -import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Base64; import java.util.EnumSet; @@ -42,7 +36,6 @@ import java.util.List; import java.util.Map; import net.snowflake.client.core.HttpUtil; -import net.snowflake.client.core.ObjectMapperFactory; import net.snowflake.client.core.SFBaseSession; import net.snowflake.client.core.SFSession; import net.snowflake.client.core.SFSessionProperty; @@ -81,8 +74,9 @@ public class SnowflakeAzureClient implements SnowflakeStorageClient { private OperationContext opContext = null; private SFBaseSession session; + private StorageClientHelper storageClientHelper; + private SnowflakeAzureClient() {} - ; /* * Factory method for a SnowflakeAzureClient object @@ -161,6 +155,7 @@ private void setupAzureClient( } catch (URISyntaxException ex) { throw new IllegalArgumentException("invalid_azure_credentials"); } + storageClientHelper = new StorageClientHelper(this, encMat, session, stageInfo.getCiphers()); } // Returns the Max number of retry attempts @@ -349,26 +344,17 @@ public void download( // Get the user-defined BLOB metadata Map userDefinedMetadata = blob.getMetadata(); - AbstractMap.SimpleEntry encryptionData = - parseEncryptionData(userDefinedMetadata.get(AZ_ENCRYPTIONDATAPROP), queryId); - - String key = encryptionData.getKey(); - String iv = encryptionData.getValue(); + DecryptionHelper decryptionHelper = + storageClientHelper.parseEncryptionDataFromJson( + userDefinedMetadata.get(AZ_ENCRYPTIONDATAPROP), queryId); if (this.isEncrypting() && this.getEncryptionKeySize() <= 256) { stopwatch.restart(); - if (key == null || iv == null) { - throw new SnowflakeSQLLoggedException( - queryId, - session, - ErrorCode.INTERNAL_ERROR.getMessageCode(), - SqlState.INTERNAL_ERROR, - "File metadata incomplete"); - } + decryptionHelper.validate(); // Decrypt file try { - EncryptionProvider.decrypt(localFile, key, iv, this.encMat); + decryptionHelper.decryptFile(localFile, encMat); stopwatch.stop(); long decryptMillis = stopwatch.elapsedMillis(); logger.info( @@ -449,26 +435,15 @@ public InputStream downloadToStream( long downloadMillis = stopwatch.elapsedMillis(); Map userDefinedMetadata = blob.getMetadata(); - AbstractMap.SimpleEntry encryptionData = - parseEncryptionData(userDefinedMetadata.get(AZ_ENCRYPTIONDATAPROP), queryId); - - String key = encryptionData.getKey(); - - String iv = encryptionData.getValue(); + DecryptionHelper decryptionHelper = + storageClientHelper.parseEncryptionDataFromJson( + userDefinedMetadata.get(AZ_ENCRYPTIONDATAPROP), queryId); if (this.isEncrypting() && this.getEncryptionKeySize() <= 256) { + decryptionHelper.validate(); stopwatch.restart(); - if (key == null || iv == null) { - throw new SnowflakeSQLLoggedException( - queryId, - session, - ErrorCode.INTERNAL_ERROR.getMessageCode(), - SqlState.INTERNAL_ERROR, - "File metadata incomplete"); - } - try { - InputStream is = EncryptionProvider.decryptStream(stream, key, iv, encMat); + InputStream is = decryptionHelper.decryptStream(stream, encMat); stopwatch.stop(); long decryptMillis = stopwatch.elapsedMillis(); logger.info( @@ -694,7 +669,7 @@ private SFPair createUploadStream( final InputStream stream; FileInputStream srcFileStream = null; try { - if (isEncrypting() && getEncryptionKeySize() < 256) { + if (isEncrypting() && getEncryptionKeySize() <= 256) { try { final InputStream uploadStream = uploadFromStream @@ -705,9 +680,7 @@ private SFPair createUploadStream( toClose.add(srcFileStream); // Encrypt - stream = - EncryptionProvider.encrypt( - meta, originalContentLength, uploadStream, this.encMat, this); + stream = storageClientHelper.encrypt(meta, originalContentLength, uploadStream); uploadFromStream = true; } catch (Exception ex) { logger.error("Failed to encrypt input", ex); @@ -934,51 +907,6 @@ private static URI buildAzureStorageEndpointURI(String storageEndPoint, String s return storageEndpoint; } - /* - * buildEncryptionMetadataJSON - * Takes the base64-encoded iv and key and creates the JSON block to be - * used as the encryptiondata metadata field on the blob. - */ - private String buildEncryptionMetadataJSON(String iv64, String key64) { - return String.format( - "{\"EncryptionMode\":\"FullBlob\",\"WrappedContentKey\"" - + ":{\"KeyId\":\"symmKey1\",\"EncryptedKey\":\"%s\"" - + ",\"Algorithm\":\"AES_CBC_256\"},\"EncryptionAgent\":" - + "{\"Protocol\":\"1.0\",\"EncryptionAlgorithm\":" - + "\"AES_CBC_256\"},\"ContentEncryptionIV\":\"%s\"" - + ",\"KeyWrappingMetadata\":{\"EncryptionLibrary\":" - + "\"Java 5.3.0\"}}", - key64, iv64); - } - - /* - * parseEncryptionData - * Takes the json string in the encryptiondata metadata field of the encrypted - * blob and parses out the key and iv. Returns the pair as key = key, iv = value. - */ - private SimpleEntry parseEncryptionData(String jsonEncryptionData, String queryId) - throws SnowflakeSQLException { - ObjectMapper mapper = ObjectMapperFactory.getObjectMapper(); - JsonFactory factory = mapper.getFactory(); - try { - JsonParser parser = factory.createParser(jsonEncryptionData); - JsonNode encryptionDataNode = mapper.readTree(parser); - - String iv = encryptionDataNode.get("ContentEncryptionIV").asText(); - String key = encryptionDataNode.get("WrappedContentKey").get("EncryptedKey").asText(); - - return new SimpleEntry(key, iv); - } catch (Exception ex) { - throw new SnowflakeSQLLoggedException( - queryId, - session, - SqlState.SYSTEM_ERROR, - ErrorCode.IO_ERROR.getMessageCode(), - ex, - "Error parsing encryption data as json" + ": " + ex.getMessage()); - } - } - /** Returns the material descriptor key */ @Override public String getMatdescKey() { @@ -996,12 +924,34 @@ public void addEncryptionMetadata( meta.addUserMetadata(getMatdescKey(), matDesc.toString()); meta.addUserMetadata( AZ_ENCRYPTIONDATAPROP, - buildEncryptionMetadataJSON( + storageClientHelper.buildEncryptionMetadataJSONForEcbCbc( Base64.getEncoder().encodeToString(ivData), Base64.getEncoder().encodeToString(encryptedKey))); meta.setContentLength(contentLength); } + @Override + public void addEncryptionMetadataForGcm( + StorageObjectMetadata meta, + MatDesc matDesc, + byte[] encryptedKey, + byte[] dataIvBytes, + byte[] keyIvBytes, + byte[] keyAad, + byte[] dataAad, + long contentLength) { + meta.addUserMetadata(getMatdescKey(), matDesc.toString()); + meta.addUserMetadata( + AZ_ENCRYPTIONDATAPROP, + storageClientHelper.buildEncryptionMetadataJSONForGcm( + Base64.getEncoder().encodeToString(keyIvBytes), + Base64.getEncoder().encodeToString(encryptedKey), + Base64.getEncoder().encodeToString(dataIvBytes), + Base64.getEncoder().encodeToString(keyAad), + Base64.getEncoder().encodeToString(dataAad))); + meta.setContentLength(contentLength); + } + /** Adds digest metadata to the StorageObjectMetadata object */ @Override public void addDigestMetadata(StorageObjectMetadata meta, String digest) { diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java index d907973ac..5c936a248 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java @@ -6,10 +6,6 @@ import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED; import static net.snowflake.client.jdbc.SnowflakeUtil.systemGetProperty; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.gax.paging.Page; import com.google.api.gax.rpc.FixedHeaderProvider; import com.google.auth.oauth2.AccessToken; @@ -33,7 +29,6 @@ import java.net.URISyntaxException; import java.nio.channels.Channels; import java.security.InvalidKeyException; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -42,7 +37,6 @@ import net.snowflake.client.core.ExecTimeTelemetryData; import net.snowflake.client.core.HttpClientSettingsKey; import net.snowflake.client.core.HttpUtil; -import net.snowflake.client.core.ObjectMapperFactory; import net.snowflake.client.core.SFSession; import net.snowflake.client.core.SFSessionProperty; import net.snowflake.client.core.SnowflakeJdbcInternalApi; @@ -78,6 +72,7 @@ * @author ppaulus */ public class SnowflakeGCSClient implements SnowflakeStorageClient { + @SnowflakeJdbcInternalApi public static final String DISABLE_GCS_DEFAULT_CREDENTIALS_PROPERTY_NAME = "net.snowflake.jdbc.disableGcsDefaultCredentials"; @@ -93,6 +88,7 @@ public class SnowflakeGCSClient implements SnowflakeStorageClient { private RemoteStoreFileEncryptionMaterial encMat; private Storage gcsClient = null; private SFSession session = null; + private StorageClientHelper storageClientHelper; private static final SFLogger logger = SFLoggerFactory.getLogger(SnowflakeGCSClient.class); @@ -262,8 +258,7 @@ public void download( File localFile = new File(localFilePath); do { try { - String key = null; - String iv = null; + DecryptionHelper decryptionHelper = null; long downloadMillis = 0; if (!Strings.isNullOrEmpty(presignedUrl)) { logger.debug("Starting download with presigned URL", false); @@ -314,11 +309,8 @@ public void download( if (header .getName() .equalsIgnoreCase(GCS_METADATA_PREFIX + GCS_ENCRYPTIONDATAPROP)) { - AbstractMap.SimpleEntry encryptionData = - parseEncryptionData(header.getValue(), queryId); - - key = encryptionData.getKey(); - iv = encryptionData.getValue(); + decryptionHelper = + storageClientHelper.parseEncryptionDataFromJson(header.getValue(), queryId); break; } } @@ -358,32 +350,19 @@ public void download( Map userDefinedMetadata = blob.getMetadata(); if (isEncrypting()) { if (userDefinedMetadata != null) { - AbstractMap.SimpleEntry encryptionData = - parseEncryptionData(userDefinedMetadata.get(GCS_ENCRYPTIONDATAPROP), queryId); - - key = encryptionData.getKey(); - iv = encryptionData.getValue(); + decryptionHelper = + storageClientHelper.parseEncryptionDataFromJson( + userDefinedMetadata.get(GCS_ENCRYPTIONDATAPROP), queryId); } } } - if (!Strings.isNullOrEmpty(iv) - && !Strings.isNullOrEmpty(key) - && this.isEncrypting() - && this.getEncryptionKeySize() <= 256) { - if (key == null || iv == null) { - throw new SnowflakeSQLLoggedException( - queryId, - session, - ErrorCode.INTERNAL_ERROR.getMessageCode(), - SqlState.INTERNAL_ERROR, - "File metadata incomplete"); - } - + if (decryptionHelper != null && this.isEncrypting() && this.getEncryptionKeySize() <= 256) { // Decrypt file + decryptionHelper.validate(); try { stopwatch.start(); - EncryptionProvider.decrypt(localFile, key, iv, this.encMat); + decryptionHelper.decryptFile(localFile, this.encMat); stopwatch.stop(); long decryptMillis = stopwatch.elapsedMillis(); logger.info( @@ -459,9 +438,7 @@ public InputStream downloadToStream( long downloadMillis = 0; do { try { - String key = null; - String iv = null; - + DecryptionHelper decryptionHelper = null; if (!Strings.isNullOrEmpty(presignedUrl)) { logger.debug("Starting download with presigned URL", false); URIBuilder uriBuilder = new URIBuilder(presignedUrl); @@ -503,11 +480,8 @@ public InputStream downloadToStream( if (header .getName() .equalsIgnoreCase(GCS_METADATA_PREFIX + GCS_ENCRYPTIONDATAPROP)) { - AbstractMap.SimpleEntry encryptionData = - parseEncryptionData(header.getValue(), queryId); - - key = encryptionData.getKey(); - iv = encryptionData.getValue(); + decryptionHelper = + storageClientHelper.parseEncryptionDataFromJson(header.getValue(), queryId); break; } } @@ -539,11 +513,9 @@ public InputStream downloadToStream( if (isEncrypting()) { // Get the user-defined BLOB metadata Map userDefinedMetadata = blob.getMetadata(); - AbstractMap.SimpleEntry encryptionData = - parseEncryptionData(userDefinedMetadata.get(GCS_ENCRYPTIONDATAPROP), queryId); - - key = encryptionData.getKey(); - iv = encryptionData.getValue(); + decryptionHelper = + storageClientHelper.parseEncryptionDataFromJson( + userDefinedMetadata.get(GCS_ENCRYPTIONDATAPROP), queryId); } stopwatch.stop(); downloadMillis = stopwatch.elapsedMillis(); @@ -551,19 +523,13 @@ public InputStream downloadToStream( if (this.isEncrypting() && this.getEncryptionKeySize() <= 256) { stopwatch.restart(); - if (key == null || iv == null) { - throw new SnowflakeSQLException( - queryId, - SqlState.INTERNAL_ERROR, - ErrorCode.INTERNAL_ERROR.getMessageCode(), - "File metadata incomplete"); - } - // Decrypt file try { if (inputStream != null) { - inputStream = EncryptionProvider.decryptStream(inputStream, key, iv, this.encMat); + if (inputStream != null && decryptionHelper != null) { + inputStream = decryptionHelper.decryptStream(inputStream, encMat); + } stopwatch.stop(); long decryptMillis = stopwatch.elapsedMillis(); logger.info( @@ -1039,7 +1005,7 @@ private SFPair createUploadStream( final InputStream stream; FileInputStream srcFileStream = null; try { - if (isEncrypting() && getEncryptionKeySize() < 256) { + if (isEncrypting() && getEncryptionKeySize() <= 256) { try { final InputStream uploadStream = uploadFromStream @@ -1050,9 +1016,7 @@ private SFPair createUploadStream( toClose.add(srcFileStream); // Encrypt - stream = - EncryptionProvider.encrypt( - meta, originalContentLength, uploadStream, this.encMat, this); + stream = storageClientHelper.encrypt(meta, originalContentLength, uploadStream); uploadFromStream = true; } catch (Exception ex) { logger.error("Failed to encrypt input", ex); @@ -1225,54 +1189,32 @@ public void addEncryptionMetadata( meta.addUserMetadata(getMatdescKey(), matDesc.toString()); meta.addUserMetadata( GCS_ENCRYPTIONDATAPROP, - buildEncryptionMetadataJSON( + storageClientHelper.buildEncryptionMetadataJSONForEcbCbc( Base64.getEncoder().encodeToString(ivData), Base64.getEncoder().encodeToString(encryptedKey))); meta.setContentLength(contentLength); } - /* - * buildEncryptionMetadataJSON - * Takes the base64-encoded iv and key and creates the JSON block to be - * used as the encryptiondata metadata field on the blob. - */ - private String buildEncryptionMetadataJSON(String iv64, String key64) { - return String.format( - "{\"EncryptionMode\":\"FullBlob\",\"WrappedContentKey\"" - + ":{\"KeyId\":\"symmKey1\",\"EncryptedKey\":\"%s\"" - + ",\"Algorithm\":\"AES_CBC_256\"},\"EncryptionAgent\":" - + "{\"Protocol\":\"1.0\",\"EncryptionAlgorithm\":" - + "\"AES_CBC_256\"},\"ContentEncryptionIV\":\"%s\"" - + ",\"KeyWrappingMetadata\":{\"EncryptionLibrary\":" - + "\"Java 5.3.0\"}}", - key64, iv64); - } - - /* - * parseEncryptionData - * Takes the json string in the encryptiondata metadata field of the encrypted - * blob and parses out the key and iv. Returns the pair as key = key, iv = value. - */ - private AbstractMap.SimpleEntry parseEncryptionData( - String jsonEncryptionData, String queryId) throws SnowflakeSQLException { - ObjectMapper mapper = ObjectMapperFactory.getObjectMapper(); - JsonFactory factory = mapper.getFactory(); - try { - JsonParser parser = factory.createParser(jsonEncryptionData); - JsonNode encryptionDataNode = mapper.readTree(parser); - - String iv = encryptionDataNode.get("ContentEncryptionIV").asText(); - String key = encryptionDataNode.get("WrappedContentKey").get("EncryptedKey").asText(); - - return new AbstractMap.SimpleEntry<>(key, iv); - } catch (Exception ex) { - throw new SnowflakeSQLException( - queryId, - ex, - SqlState.SYSTEM_ERROR, - ErrorCode.IO_ERROR.getMessageCode(), - "Error parsing encryption data as json" + ": " + ex.getMessage()); - } + @Override + public void addEncryptionMetadataForGcm( + StorageObjectMetadata meta, + MatDesc matDesc, + byte[] encryptedKey, + byte[] dataIvBytes, + byte[] keyIvBytes, + byte[] keyAad, + byte[] dataAad, + long contentLength) { + meta.addUserMetadata(getMatdescKey(), matDesc.toString()); + meta.addUserMetadata( + GCS_ENCRYPTIONDATAPROP, + storageClientHelper.buildEncryptionMetadataJSONForGcm( + Base64.getEncoder().encodeToString(keyIvBytes), + Base64.getEncoder().encodeToString(encryptedKey), + Base64.getEncoder().encodeToString(dataIvBytes), + Base64.getEncoder().encodeToString(keyAad), + Base64.getEncoder().encodeToString(dataAad))); + meta.setContentLength(contentLength); } /** Adds digest metadata to the StorageObjectMetadata object */ @@ -1351,6 +1293,8 @@ private void setupGCSClient( } catch (Exception ex) { throw new IllegalArgumentException("invalid_gcs_credentials"); } + this.storageClientHelper = + new StorageClientHelper(this, encMat, session, stageInfo.getCiphers()); } private static boolean areDisabledGcsDefaultCredentials(SFSession session) { diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeS3Client.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeS3Client.java index 3b33b60f0..f64646433 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeS3Client.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeS3Client.java @@ -84,8 +84,12 @@ public class SnowflakeS3Client implements SnowflakeStorageClient { private static final SFLogger logger = SFLoggerFactory.getLogger(SnowflakeS3Client.class); private static final String localFileSep = systemGetProperty("file.separator"); private static final String AES = "AES"; - private static final String AMZ_KEY = "x-amz-key"; - private static final String AMZ_IV = "x-amz-iv"; + static final String AMZ_KEY = "x-amz-key"; + static final String AMZ_DATA_IV = "x-amz-iv"; + static final String AMZ_KEY_IV = "x-amz-key-iv"; + static final String AMZ_CIPHER = "x-amz-cipher"; + static final String AMZ_KEY_AAD = "x-amz-key-aad"; + static final String AMZ_DATA_AAD = "x-amz-data-aad"; private static final String S3_STREAMING_INGEST_CLIENT_NAME = "ingestclientname"; private static final String S3_STREAMING_INGEST_CLIENT_KEY = "ingestclientkey"; @@ -102,10 +106,14 @@ public class SnowflakeS3Client implements SnowflakeStorageClient { private SFBaseSession session = null; private boolean isClientSideEncrypted = true; private boolean isUseS3RegionalUrl = false; + private StageInfo.Ciphers ciphers = null; + + private StorageClientHelper storageClientHelper; // socket factory used by s3 client's http client. private static SSLConnectionSocketFactory s3ConnectionSocketFactory = null; + @Deprecated public SnowflakeS3Client( Map stageCredentials, ClientConfiguration clientConfig, @@ -117,6 +125,31 @@ public SnowflakeS3Client( SFBaseSession session, boolean useS3RegionalUrl) throws SnowflakeSQLException { + this( + stageCredentials, + clientConfig, + encMat, + proxyProperties, + stageRegion, + stageEndPoint, + isClientSideEncrypted, + session, + useS3RegionalUrl, + null); + } + + public SnowflakeS3Client( + Map stageCredentials, + ClientConfiguration clientConfig, + RemoteStoreFileEncryptionMaterial encMat, + Properties proxyProperties, + String stageRegion, + String stageEndPoint, + boolean isClientSideEncrypted, + SFBaseSession session, + boolean useS3RegionalUrl, + StageInfo.Ciphers ciphers) + throws SnowflakeSQLException { logger.debug( "Initializing Snowflake S3 client with encryption: {}, client side encrypted: {}", encMat != null, @@ -131,7 +164,8 @@ public SnowflakeS3Client( stageRegion, stageEndPoint, isClientSideEncrypted, - session); + session, + ciphers); } private void setupSnowflakeS3Client( @@ -142,7 +176,8 @@ private void setupSnowflakeS3Client( String stageRegion, String stageEndPoint, boolean isClientSideEncrypted, - SFBaseSession session) + SFBaseSession session, + StageInfo.Ciphers ciphers) throws SnowflakeSQLException { // Save the client creation parameters so that we can reuse them, // to reset the AWS client. We won't save the awsCredentials since @@ -233,7 +268,8 @@ private void setupSnowflakeS3Client( } // Explicitly force to use virtual address style amazonS3Builder.withPathStyleAccessEnabled(false); - amazonClient = (AmazonS3) amazonS3Builder.build(); + amazonClient = amazonS3Builder.build(); + storageClientHelper = new StorageClientHelper(this, encMat, session, ciphers); } static String getDomainSuffixForRegionalUrl(String regionName) { @@ -294,7 +330,8 @@ public void renew(Map stageCredentials) throws SnowflakeSQLException { this.stageRegion, this.stageEndPoint, this.isClientSideEncrypted, - this.session); + this.session, + this.ciphers); } @Override @@ -379,29 +416,20 @@ public ExecutorService newExecutor() { // Pull object metadata from S3 ObjectMetadata meta = amazonClient.getObjectMetadata(remoteStorageLocation, stageFilePath); - Map metaMap = meta.getUserMetadata(); - String key = metaMap.get(AMZ_KEY); - String iv = metaMap.get(AMZ_IV); - myDownload.waitForCompletion(); stopwatch.stop(); long downloadMillis = stopwatch.elapsedMillis(); - if (this.isEncrypting() && this.getEncryptionKeySize() < 256) { - stopwatch.restart(); - if (key == null || iv == null) { - throw new SnowflakeSQLLoggedException( - queryId, - session, - ErrorCode.INTERNAL_ERROR.getMessageCode(), - SqlState.INTERNAL_ERROR, - "File metadata incomplete"); - } + if (this.isEncrypting() && this.getEncryptionKeySize() <= 256) { + Map metaMap = meta.getUserMetadata(); + DecryptionHelper decryptionHelper = + storageClientHelper.buildEncryptionMetadataFromAwsMetadata(queryId, metaMap); + stopwatch.restart(); // Decrypt file try { - EncryptionProvider.decrypt(localFile, key, iv, this.encMat); + decryptionHelper.decryptFile(localFile, encMat); stopwatch.stop(); long decryptMillis = stopwatch.elapsedMillis(); logger.info( @@ -483,22 +511,13 @@ public InputStream downloadToStream( long downloadMillis = stopwatch.elapsedMillis(); Map metaMap = meta.getUserMetadata(); - String key = metaMap.get(AMZ_KEY); - String iv = metaMap.get(AMZ_IV); - - if (this.isEncrypting() && this.getEncryptionKeySize() < 256) { + if (this.isEncrypting() && this.getEncryptionKeySize() <= 256) { + DecryptionHelper decryptionHelper = + storageClientHelper.buildEncryptionMetadataFromAwsMetadata(queryId, metaMap); stopwatch.restart(); - if (key == null || iv == null) { - throw new SnowflakeSQLLoggedException( - queryId, - session, - ErrorCode.INTERNAL_ERROR.getMessageCode(), - SqlState.INTERNAL_ERROR, - "File metadata incomplete"); - } try { - InputStream is = EncryptionProvider.decryptStream(stream, key, iv, encMat); + InputStream is = decryptionHelper.decryptStream(stream, encMat); stopwatch.stop(); long decryptMillis = stopwatch.elapsedMillis(); logger.info( @@ -723,7 +742,7 @@ private SFPair createUploadStream( this.getEncryptionKeySize()); final InputStream result; FileInputStream srcFileStream = null; - if (isEncrypting() && getEncryptionKeySize() < 256) { + if (isEncrypting() && getEncryptionKeySize() <= 256) { try { final InputStream uploadStream = uploadFromStream @@ -735,9 +754,7 @@ private SFPair createUploadStream( // Encrypt S3StorageObjectMetadata s3Metadata = new S3StorageObjectMetadata(meta); - result = - EncryptionProvider.encrypt( - s3Metadata, originalContentLength, uploadStream, this.encMat, this); + result = storageClientHelper.encrypt(s3Metadata, originalContentLength, inputStream); uploadFromStream = true; } catch (Exception ex) { logger.error("Failed to encrypt input", ex); @@ -966,7 +983,28 @@ public void addEncryptionMetadata( long contentLength) { meta.addUserMetadata(getMatdescKey(), matDesc.toString()); meta.addUserMetadata(AMZ_KEY, Base64.getEncoder().encodeToString(encryptedKey)); - meta.addUserMetadata(AMZ_IV, Base64.getEncoder().encodeToString(ivData)); + meta.addUserMetadata(AMZ_DATA_IV, Base64.getEncoder().encodeToString(ivData)); + meta.addUserMetadata(AMZ_CIPHER, "AES_CBC"); + meta.setContentLength(contentLength); + } + + @Override + public void addEncryptionMetadataForGcm( + StorageObjectMetadata meta, + MatDesc matDesc, + byte[] encryptedKey, + byte[] dataIvBytes, + byte[] keyIvBytes, + byte[] keyAad, + byte[] dataAad, + long contentLength) { + meta.addUserMetadata(getMatdescKey(), matDesc.toString()); + meta.addUserMetadata(AMZ_KEY, Base64.getEncoder().encodeToString(encryptedKey)); + meta.addUserMetadata(AMZ_KEY_IV, Base64.getEncoder().encodeToString(keyIvBytes)); + meta.addUserMetadata(AMZ_KEY_AAD, Base64.getEncoder().encodeToString(keyAad)); + meta.addUserMetadata(AMZ_DATA_IV, Base64.getEncoder().encodeToString(dataIvBytes)); + meta.addUserMetadata(AMZ_DATA_AAD, Base64.getEncoder().encodeToString(dataAad)); + meta.addUserMetadata(AMZ_CIPHER, "AES_GCM"); meta.setContentLength(contentLength); } diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeStorageClient.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeStorageClient.java index 4be936763..42b21858b 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeStorageClient.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeStorageClient.java @@ -485,7 +485,7 @@ void addEncryptionMetadata( * @param contentLength the length of the encrypted content */ @SnowflakeJdbcInternalApi - default void addEncryptionMetadataForGcm( + void addEncryptionMetadataForGcm( StorageObjectMetadata meta, MatDesc matDesc, byte[] encryptedKey, @@ -493,9 +493,7 @@ default void addEncryptionMetadataForGcm( byte[] keyIvBytes, byte[] keyAad, byte[] dataAad, - long contentLength) { - // TODO GCM SNOW-1431870 - } + long contentLength); /** * Adds digest metadata to the StorageObjectMetadata object diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java index 7a8bf4d36..ac8827d70 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java @@ -3,9 +3,15 @@ import java.io.Serializable; import java.util.Map; import java.util.Properties; +import net.snowflake.client.core.Event; +import net.snowflake.client.core.SnowflakeJdbcInternalApi; +import net.snowflake.client.log.SFLogger; +import net.snowflake.client.log.SFLoggerFactory; /** Encapsulates all the required stage properties used by GET/PUT for Azure and S3 stages */ public class StageInfo implements Serializable { + private static final SFLogger logger = SFLoggerFactory.getLogger(Event.class); + public enum StageType { S3, AZURE, @@ -13,6 +19,13 @@ public enum StageType { GCS } + // First value represents key encryption mode, second value represents file content encryption + // mode. + public enum Ciphers { + AESECB_AESCBC, + AESGCM_AESGCM + } + private static final long serialVersionUID = 1L; private StageType stageType; // The stage type private String location; // The container or bucket @@ -22,9 +35,34 @@ public enum StageType { private String storageAccount; // The Azure Storage account (Azure only) private String presignedUrl; // GCS gives us back a presigned URL instead of a cred private boolean isClientSideEncrypted; // whether to encrypt/decrypt files on the stage + private Ciphers ciphers; private boolean useS3RegionalUrl; // whether to use s3 regional URL (AWS Only) private Properties proxyProperties; + /* + * @deprecated Use {@link #createStageInfo(String, String, Map, String, String, String, boolean, String)} + */ + @Deprecated + public static StageInfo createStageInfo( + String locationType, + String location, + Map credentials, + String region, + String endPoint, + String storageAccount, + boolean isClientSideEncrypted) + throws IllegalArgumentException { + return createStageInfo( + locationType, + location, + credentials, + region, + endPoint, + storageAccount, + isClientSideEncrypted, + null); + } + /* * Creates a StageInfo object * Validates that the necessary Stage info arguments are specified @@ -38,6 +76,7 @@ public enum StageType { * @param isClientSideEncrypted Whether the stage should use client-side encryption * @throws IllegalArgumentException one or more parameters required were missing */ + @SnowflakeJdbcInternalApi public static StageInfo createStageInfo( String locationType, String location, @@ -45,8 +84,8 @@ public static StageInfo createStageInfo( String region, String endPoint, String storageAccount, - boolean isClientSideEncrypted) - throws IllegalArgumentException { + boolean isClientSideEncrypted, + String ciphersString) { StageType stageType; // Ensure that all the required parameters are specified switch (locationType) { @@ -84,8 +123,28 @@ public static StageInfo createStageInfo( default: throw new IllegalArgumentException("Invalid stage type: " + locationType); } + Ciphers ciphers = null; + if (isClientSideEncrypted) { + if (ciphersString == null || ciphersString.isEmpty()) { + logger.debug("No ciphers specified for stage, using ECB/CBC mode"); + ciphers = Ciphers.AESECB_AESCBC; + } else if (ciphersString.equals("AES_CBC")) { + ciphers = Ciphers.AESECB_AESCBC; + } else if (ciphersString.equals("AES_GCM") || ciphersString.equals("AES_GCM,AES_CBC")) { + ciphers = Ciphers.AESGCM_AESGCM; + } else { + throw new IllegalArgumentException("Invalid ciphers: " + ciphersString); + } + } return new StageInfo( - stageType, location, credentials, region, endPoint, storageAccount, isClientSideEncrypted); + stageType, + location, + credentials, + region, + endPoint, + storageAccount, + isClientSideEncrypted, + ciphers); } /* @@ -108,7 +167,8 @@ private StageInfo( String region, String endPoint, String storageAccount, - boolean isClientSideEncrypted) { + boolean isClientSideEncrypted, + Ciphers ciphers) { this.stageType = stageType; this.location = location; this.credentials = credentials; @@ -116,6 +176,7 @@ private StageInfo( this.endPoint = endPoint; this.storageAccount = storageAccount; this.isClientSideEncrypted = isClientSideEncrypted; + this.ciphers = ciphers; } public StageType getStageType() { @@ -158,6 +219,10 @@ public boolean getIsClientSideEncrypted() { return isClientSideEncrypted; } + public Ciphers getCiphers() { + return ciphers; + } + public void setUseS3RegionalUrl(boolean useS3RegionalUrl) { this.useS3RegionalUrl = useS3RegionalUrl; } diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java index ac7de73a6..39e33f0a1 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java @@ -69,7 +69,8 @@ public SnowflakeStorageClient createClient( stage.getEndPoint(), stage.getIsClientSideEncrypted(), session, - useS3RegionalUrl); + useS3RegionalUrl, + stage.getCiphers()); case AZURE: return createAzureClient(stage, encMat, session); @@ -109,7 +110,8 @@ private SnowflakeS3Client createS3Client( String stageEndPoint, boolean isClientSideEncrypted, SFBaseSession session, - boolean useS3RegionalUrl) + boolean useS3RegionalUrl, + StageInfo.Ciphers ciphers) throws SnowflakeSQLException { final int S3_TRANSFER_MAX_RETRIES = 3; @@ -148,7 +150,8 @@ private SnowflakeS3Client createS3Client( stageEndPoint, isClientSideEncrypted, session, - useS3RegionalUrl); + useS3RegionalUrl, + ciphers); } catch (Exception ex) { logger.debug("Exception creating s3 client", ex); throw ex; diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientHelper.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientHelper.java new file mode 100644 index 000000000..37e2e934e --- /dev/null +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientHelper.java @@ -0,0 +1,177 @@ +package net.snowflake.client.jdbc.cloud.storage; + +import static net.snowflake.client.jdbc.cloud.storage.SnowflakeS3Client.AMZ_CIPHER; +import static net.snowflake.client.jdbc.cloud.storage.SnowflakeS3Client.AMZ_DATA_AAD; +import static net.snowflake.client.jdbc.cloud.storage.SnowflakeS3Client.AMZ_DATA_IV; +import static net.snowflake.client.jdbc.cloud.storage.SnowflakeS3Client.AMZ_KEY; +import static net.snowflake.client.jdbc.cloud.storage.SnowflakeS3Client.AMZ_KEY_AAD; +import static net.snowflake.client.jdbc.cloud.storage.SnowflakeS3Client.AMZ_KEY_IV; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.util.Map; +import java.util.Optional; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import net.snowflake.client.core.ObjectMapperFactory; +import net.snowflake.client.core.SFBaseSession; +import net.snowflake.client.jdbc.ErrorCode; +import net.snowflake.client.jdbc.SnowflakeSQLException; +import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; +import net.snowflake.common.core.RemoteStoreFileEncryptionMaterial; +import net.snowflake.common.core.SqlState; + +class StorageClientHelper { + private static final byte[] keyAad = "".getBytes(StandardCharsets.UTF_8); + private static final byte[] dataAad = "".getBytes(StandardCharsets.UTF_8); + + private final SnowflakeStorageClient client; + private final RemoteStoreFileEncryptionMaterial encMat; + private final SFBaseSession session; + private final StageInfo.Ciphers ciphers; + + StorageClientHelper( + SnowflakeStorageClient client, + RemoteStoreFileEncryptionMaterial encMat, + SFBaseSession session, + StageInfo.Ciphers ciphers) { + this.client = client; + this.encMat = encMat; + this.session = session; + this.ciphers = ciphers; + } + + String buildEncryptionMetadataJSONForEcbCbc(String iv64, String key64) { + return String.format( + "{\"EncryptionMode\":\"FullBlob\",\"WrappedContentKey\"" + + ":{\"KeyId\":\"symmKey1\",\"EncryptedKey\":\"%s\"" + + ",\"Algorithm\":\"AES_CBC_256\"},\"EncryptionAgent\":" + + "{\"Protocol\":\"1.0\",\"EncryptionAlgorithm\":" + + "\"AES_CBC_256\"},\"ContentEncryptionIV\":\"%s\"" + + ",\"KeyWrappingMetadata\":{\"EncryptionLibrary\":" + + "\"Java 5.3.0\"}}", + key64, iv64); + } + + String buildEncryptionMetadataJSONForGcm( + String keyIv, String key, String contentIv, String fileKeyAad, String contentAad) { + return String.format( + "{" + + "\"EncryptionMode\":\"FullBlob\"," + + "\"WrappedContentKey\":{" + + "\"KeyId\":\"symmKey1\"," + + "\"EncryptedKey\":\"%s\"," + + "\"Algorithm\":\"AES_GCM\"," + + "\"KeyEncryptionIV\":\"%s\"," + + "\"FileKeyAad\":\"%s\"" + + "}," + + "\"EncryptionAgent\":{" + + "\"Protocol\":\"1.0\"," + + "\"EncryptionAlgorithm\":\"AES_GCM\"" + + "}," + + "\"ContentEncryptionIV\":\"%s\"," + + "\"ContentAad\":\"%s\"," + + "\"KeyWrappingMetadata\":{" + + "\"EncryptionLibrary\":\"Java 5.3.0\"" + + "}" + + "}", + key, keyIv, fileKeyAad, contentIv, contentAad); + } + + DecryptionHelper buildEncryptionMetadataFromAwsMetadata( + String queryId, Map metaMap) throws SnowflakeSQLLoggedException { + StageInfo.Ciphers ciphers = + Optional.ofNullable(metaMap.get(AMZ_CIPHER)) + .map(this::toCiphers) + .orElse(StageInfo.Ciphers.AESECB_AESCBC); + String key = metaMap.get(AMZ_KEY); + String keyIv = metaMap.get(AMZ_KEY_IV); + String dataIv = metaMap.get(AMZ_DATA_IV); + String keyAad = metaMap.get(AMZ_KEY_AAD); + String dataAad = metaMap.get(AMZ_DATA_AAD); + switch (ciphers) { + case AESECB_AESCBC: + return DecryptionHelper.forCbc(queryId, session, key, dataIv); + case AESGCM_AESGCM: + return DecryptionHelper.forGcm(queryId, session, key, keyIv, dataIv, keyAad, dataAad); + } + throw new IllegalArgumentException("unknown cipher " + ciphers); + } + + private StageInfo.Ciphers toCiphers(String s) { + switch (s) { + case "AES_CBC": + case "": + return StageInfo.Ciphers.AESECB_AESCBC; + case "AES_GCM": + return StageInfo.Ciphers.AESGCM_AESGCM; + } + throw new IllegalArgumentException("Unknown cipher " + s); + } + + DecryptionHelper parseEncryptionDataFromJson(String jsonEncryptionData, String queryId) + throws SnowflakeSQLException { + ObjectMapper mapper = ObjectMapperFactory.getObjectMapper(); + JsonFactory factory = mapper.getFactory(); + try { + JsonParser parser = factory.createParser(jsonEncryptionData); + JsonNode encryptionDataNode = mapper.readTree(parser); + + String contentIv = encryptionDataNode.get("ContentEncryptionIV").asText(); + String contentAad = + Optional.ofNullable(encryptionDataNode.get("ContentAad")) + .map(JsonNode::asText) + .orElse(null); + String key = encryptionDataNode.get("WrappedContentKey").get("EncryptedKey").asText(); + String keyIv = + Optional.ofNullable(encryptionDataNode.get("WrappedContentKey")) + .map(v -> v.get("KeyEncryptionIV")) + .map(JsonNode::asText) + .orElse(null); + String fileKeyAad = + Optional.ofNullable(encryptionDataNode.get("WrappedContentKey")) + .map(v -> v.get("FileKeyAad")) + .map(JsonNode::asText) + .orElse(null); + + String algorithm = encryptionDataNode.get("WrappedContentKey").get("Algorithm").asText(); + if (algorithm.contains("AES_GCM")) { + return DecryptionHelper.forGcm( + queryId, session, key, keyIv, contentIv, fileKeyAad, contentAad); + } else { + return DecryptionHelper.forCbc(queryId, session, key, contentIv); + } + + } catch (Exception ex) { + throw new SnowflakeSQLException( + queryId, + ex, + SqlState.SYSTEM_ERROR, + ErrorCode.IO_ERROR.getMessageCode(), + "Error parsing encryption data as json" + ": " + ex.getMessage()); + } + } + + InputStream encrypt( + StorageObjectMetadata meta, long originalContentLength, InputStream uploadStream) + throws InvalidKeyException, InvalidAlgorithmParameterException, NoSuchAlgorithmException, + NoSuchProviderException, NoSuchPaddingException, FileNotFoundException, + IllegalBlockSizeException, BadPaddingException { + if (ciphers == null || ciphers == StageInfo.Ciphers.AESECB_AESCBC) { + return EncryptionProvider.encrypt(meta, originalContentLength, uploadStream, encMat, client); + } else { + return GcmEncryptionProvider.encrypt( + meta, originalContentLength, uploadStream, encMat, client, dataAad, keyAad); + } + } +} diff --git a/src/test/java/net/snowflake/client/AbstractDriverIT.java b/src/test/java/net/snowflake/client/AbstractDriverIT.java index 4a3acea23..86c99fcff 100644 --- a/src/test/java/net/snowflake/client/AbstractDriverIT.java +++ b/src/test/java/net/snowflake/client/AbstractDriverIT.java @@ -327,6 +327,10 @@ public static Connection getConnection( properties.put("internal", Boolean.TRUE.toString()); // TODO: do we need this? properties.put("insecureMode", false); // use OCSP for all tests. + // properties.put("useProxy", "true"); + // properties.put("proxyHost", "localhost"); + // properties.put("proxyPort", "8080"); + if (injectSocketTimeout > 0) { properties.put("injectSocketTimeout", String.valueOf(injectSocketTimeout)); } diff --git a/src/test/java/net/snowflake/client/jdbc/cloud/storage/CloudStorageClientLatestIT.java b/src/test/java/net/snowflake/client/jdbc/cloud/storage/CloudStorageClientLatestIT.java index 20a070a02..211f63609 100644 --- a/src/test/java/net/snowflake/client/jdbc/cloud/storage/CloudStorageClientLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/cloud/storage/CloudStorageClientLatestIT.java @@ -12,11 +12,14 @@ import net.snowflake.client.category.TestCategoryOthers; import net.snowflake.client.jdbc.BaseJDBCTest; import net.snowflake.client.jdbc.SnowflakeConnection; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; @Category(TestCategoryOthers.class) public class CloudStorageClientLatestIT extends BaseJDBCTest { + @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); /** * Test for SNOW-565154 - it was waiting for ~5 minutes so the test is waiting much shorter time