From b19db8afcecf7b8bf8f165d6e1d03a16bc7382d2 Mon Sep 17 00:00:00 2001 From: zencircle Date: Wed, 30 Oct 2024 16:28:27 -0400 Subject: [PATCH] add sigv4 config for dev --- build.sbt | 3 +- .../resources/persistence-keyspace-sigv4.conf | 108 ++++++++++++++++++ .../scala/hmda/auth/OAuth2Authorization.scala | 2 +- .../application-dev-keyspace-sigv4.conf | 13 +++ .../resources/application-dev-keyspace.conf | 2 +- hmda/src/main/scala/hmda/HmdaPlatform.scala | 5 +- project/Dependencies.scala | 1 + 7 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 common/src/main/resources/persistence-keyspace-sigv4.conf create mode 100644 hmda/src/main/resources/application-dev-keyspace-sigv4.conf diff --git a/build.sbt b/build.sbt index e32debb50c..bff26308c5 100644 --- a/build.sbt +++ b/build.sbt @@ -46,6 +46,7 @@ lazy val akkaPersistenceDeps = akkaPersistenceQuery, akkaClusterShardingTyped, akkaPersistenceCassandra, + keyspacedriver, cassandraLauncher ) @@ -176,7 +177,7 @@ lazy val `hmda-platform` = (project in file("hmda")) val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) }, - reStart / envVars ++= Map("CASSANDRA_CLUSTER_HOSTS" -> "localhost", "APP_PORT" -> "2551"), +// reStart / envVars ++= Map("CASSANDRA_CLUSTER_HOSTS" -> "localhost", "APP_PORT" -> "2551"), ), dockerSettings, packageSettings diff --git a/common/src/main/resources/persistence-keyspace-sigv4.conf b/common/src/main/resources/persistence-keyspace-sigv4.conf new file mode 100644 index 0000000000..0a3e1b5426 --- /dev/null +++ b/common/src/main/resources/persistence-keyspace-sigv4.conf @@ -0,0 +1,108 @@ +include "serialization.conf" + +akka { + + persistence { + journal.plugin = "akka.persistence.cassandra.journal" + snapshot-store.plugin = "akka.persistence.cassandra.snapshot" + + query { + journal.id = "akka.persistence.cassandra.query" + } + + cassandra { + events-by-tag { + max-message-batch-size = 30 + max-message-batch-size = ${?TAG_BATCH_SIZE} + flush-interval = 150ms + flush-interval = ${?TAG_FLUSH_INTERVAL} + } + journal { + keyspace = "hmda2_journal" + keyspace = ${?CASSANDRA_JOURNAL_KEYSPACE} + table = "journal" + events-by-tag.max-message-batch-size = 30 + events-by-tag.max-message-batch-size = ${?TAG_BATCH_SIZE} + events-by-tag.flush-interval = 150ms + events-by-tag.flush-interval = ${?TAG_FLUSH_INTERVAL} + } + query { + # reference: https://github.com/akka/akka-persistence-cassandra/blob/v0.101/core/src/main/resources/reference.conf + # Sequence numbers for a persistenceId is assumed to be monotonically increasing + # without gaps. That is used for detecting missing events. + # In early versions of the journal that might not be true and therefore + # this can be relaxed by setting this property to off. + gap-free-sequence-numbers = off + } + snapshot { + keyspace = "hmda2_snapshot" + keyspace = ${?CASSANDRA_SNAPSHOT_KEYSPACE} + table = "snapshot" + } + } + } + + +} + +datastax-java-driver { + profiles { + akka-persistence-cassandra-profile { + basic { + request.consistency = LOCAL_QUORUM + } + } + } + basic { + contact-points = ["localhost:9042"] + contact-points = [${?CASSANDRA_CLUSTER_HOSTS}] + load-balancing-policy.local-datacenter = "" + load-balancing-policy.local-datacenter = ${?CASSANDRA_CLUSTER_DC} + load-balancing-policy.slow-replica-avoidance = false + request.consistency = LOCAL_QUORUM + } + + + + advanced { + + advanced.protocol { + version = V4 + } + auth-provider { + class = software.aws.mcs.auth.SigV4AuthProvider + aws-region = "us-east-1" + } + request-tracker { + classes = [RequestLogger] + logs { + slow { + threshold = 1 second + enabled = true + } + } + } + ssl-engine-factory { + class = DefaultSslEngineFactory + truststore-path = ${?CASSANDRA_TRUSTSTORE_FILE} + truststore-password = ${?CASSANDRA_TRUSTSTORE_PASSWORD} + hostname-validation = false + } + + } +} + +kafka { + hosts = "localhost:9092" + hosts = ${?KAFKA_CLUSTER_HOSTS} + idle-timeout = 5 + idle-timeout = ${?KAFKA_IDLE_TIMEOUT} + security.protocol="" + security.protocol=${?KAFKA_SECURITY} + ssl.truststore.location = "" + ssl.truststore.location = ${?TRUSTSTORE_PATH} + ssl.truststore.password = "" + ssl.truststore.password = ${?TRUSTSTORE_PASSWORD} + ssl.endpoint = "" + ssl.endpoint = ${?KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG} +} \ No newline at end of file diff --git a/common/src/main/scala/hmda/auth/OAuth2Authorization.scala b/common/src/main/scala/hmda/auth/OAuth2Authorization.scala index 1679dc557d..549754ae94 100644 --- a/common/src/main/scala/hmda/auth/OAuth2Authorization.scala +++ b/common/src/main/scala/hmda/auth/OAuth2Authorization.scala @@ -118,7 +118,7 @@ class OAuth2Authorization(logger: Logger, tokenVerifier: TokenVerifier) { } protected def withLocalModeBypass(thunk: => Directive1[VerifiedToken]): Directive1[VerifiedToken] = - if (runtimeMode == "dev" || runtimeMode == "dev-keyspace" || runtimeMode == "docker-compose" || runtimeMode == "kind") { + if (runtimeMode == "dev" || runtimeMode == "dev-keyspace" || runtimeMode == "dev-keyspace-sigv4" || runtimeMode == "docker-compose" || runtimeMode == "kind") { provide(VerifiedToken()) } else { thunk } diff --git a/hmda/src/main/resources/application-dev-keyspace-sigv4.conf b/hmda/src/main/resources/application-dev-keyspace-sigv4.conf new file mode 100644 index 0000000000..1dcab5b4ec --- /dev/null +++ b/hmda/src/main/resources/application-dev-keyspace-sigv4.conf @@ -0,0 +1,13 @@ +include "application.conf" +include "persistence-keyspace-sigv4.conf" + +APP_PORT = 2551 +APP_PORT = ${?APP_PORT} + +akka { + + cluster { + seed-nodes = ["akka://hmda2@127.0.0.1:"${APP_PORT}] + } + +} \ No newline at end of file diff --git a/hmda/src/main/resources/application-dev-keyspace.conf b/hmda/src/main/resources/application-dev-keyspace.conf index 84a6585078..1dcab5b4ec 100644 --- a/hmda/src/main/resources/application-dev-keyspace.conf +++ b/hmda/src/main/resources/application-dev-keyspace.conf @@ -1,5 +1,5 @@ include "application.conf" -include "persistence-keyspace.conf" +include "persistence-keyspace-sigv4.conf" APP_PORT = 2551 APP_PORT = ${?APP_PORT} diff --git a/hmda/src/main/scala/hmda/HmdaPlatform.scala b/hmda/src/main/scala/hmda/HmdaPlatform.scala index 93a7ec3e1f..7e890d1dae 100644 --- a/hmda/src/main/scala/hmda/HmdaPlatform.scala +++ b/hmda/src/main/scala/hmda/HmdaPlatform.scala @@ -58,6 +58,9 @@ object HmdaPlatform extends App { case "dev-keyspace" => ConfigFactory.parseResources("application-dev-keyspace.conf").resolve() + case "dev-keyspace-sigv4" => + ConfigFactory.parseResources("application-dev-keyspace.conf").resolve() + case "kubernetes" => log.info(s"HOSTNAME: ${System.getenv("HOSTNAME")}") log.info(s"HOSTADDRESS: " + InetAddress.getLocalHost().getHostAddress()) @@ -80,7 +83,7 @@ object HmdaPlatform extends App { AkkaManagement(system).start() } - if (runtimeMode == "dev" || runtimeMode == "dev-keyspace") { + if (runtimeMode == "dev" || runtimeMode == "dev-keyspace" || runtimeMode == "dev-keyspace-sigv4") { CassandraUtil.startEmbeddedCassandra() AkkaManagement(system).start() implicit val embeddedKafkaConfig: EmbeddedKafkaConfig = EmbeddedKafkaConfig( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8c0ad4617b..0cba661bb3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -83,6 +83,7 @@ object Dependencies { lazy val scalacheckShapeless = "com.github.alexarchambault" %% "scalacheck-shapeless_1.14" % Version.scalacheckShapeless % Test lazy val diffx = "com.softwaremill.diffx" %% "diffx-core" % Version.diffx % Test lazy val kubernetesApi = "io.kubernetes" % "client-java" % Version.kubernetesApi + lazy val keyspacedriver = "software.aws.mcs" % "aws-sigv4-auth-cassandra-java-driver-plugin" % "4.0.9" // overriding the log4j-slf4j bridge used by spring, transitively brought in by s3mock // this is needed because of CVE-2021-44228 https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-44228