Fix tests

snowplow · Aug 21, 2024 · 13b8305 · 13b8305
1 parent 63a7aed
commit 13b8305
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 9 deletions.
diff --git a/config/loader/aws/databricks.config.reference.hocon b/config/loader/aws/databricks.config.reference.hocon
@@ -27,7 +27,7 @@
     "oauth": {
       "clientId": "client-id"
       "clientSecret": ${OAUTH_CLIENT_SECRET}
-    }
+    },
     # Optional. Override the Databricks default catalog, e.g. with a Unity catalog name.
     "catalog": "hive_metastore",
     # DB schema

diff --git a/config/loader/azure/databricks.config.reference.hocon b/config/loader/azure/databricks.config.reference.hocon
@@ -43,6 +43,10 @@
         "parameterName": "snowplow.databricks.password"
       }
     },
+   "oauth": {
+     "clientId": "client-id"
+     "clientSecret": ${OAUTH_CLIENT_SECRET}
+    },
     # Optional. Override the Databricks default catalog, e.g. with a Unity catalog name.
     "catalog": "hive_metastore",
     # DB schema

diff --git a/config/loader/gcp/databricks.config.reference.hocon b/config/loader/gcp/databricks.config.reference.hocon
@@ -21,6 +21,10 @@
         "parameterName": "snowplow.databricks.password"
       }
     },
+   "oauth": {
+     "clientId": "client-id"
+     "clientSecret": ${OAUTH_CLIENT_SECRET}
+    },    
     # Optional. Override the Databricks default catalog, e.g. with a Unity catalog name.
     "catalog": "hive_metastore",
     # DB schema

diff --git a/...s-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala b/...s-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala
@@ -112,7 +112,8 @@ class ConfigSpec extends Specification {
       val result = getConfigFromResource("/loader/aws/databricks.config.minimal.hocon", testParseConfig)
       val storage = ConfigSpec.exampleStorage.copy(
         catalog  = None,
-        password = StorageTarget.PasswordConfig.PlainText("Supersecret1")
+        password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")),
+        oauth    = None
       )
       val cloud      = Config.Cloud.AWS(RegionSpec.DefaultTestRegion, exampleMessageQueue.copy(region = Some(RegionSpec.DefaultTestRegion)))
       val retries    = exampleRetries.copy(cumulativeBound = Some(20.minutes))
@@ -140,7 +141,8 @@ class ConfigSpec extends Specification {
       val result = getConfigFromResource("/loader/gcp/databricks.config.minimal.hocon", testParseConfig)
       val storage = ConfigSpec.exampleStorage.copy(
         catalog  = None,
-        password = StorageTarget.PasswordConfig.PlainText("Supersecret1")
+        password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")),
+        oauth    = None
       )
       val retries     = exampleRetries.copy(cumulativeBound = Some(20.minutes))
       val readyCheck  = exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds)
@@ -167,7 +169,8 @@ class ConfigSpec extends Specification {
       val result = getConfigFromResource("/loader/azure/databricks.config.minimal.hocon", testParseConfig)
       val storage = ConfigSpec.exampleStorage.copy(
         catalog  = None,
-        password = StorageTarget.PasswordConfig.PlainText("Supersecret1")
+        password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")),
+        oauth    = None
       )
       val retries     = exampleRetries.copy(cumulativeBound = Some(20.minutes))
       val readyCheck  = exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds)
@@ -200,7 +203,8 @@ object ConfigSpec {
     "atomic",
     443,
     "/databricks/http/path",
-    StorageTarget.PasswordConfig.EncryptedKey(StorageTarget.EncryptedConfig("snowplow.databricks.password")),
+    Some(StorageTarget.PasswordConfig.EncryptedKey(StorageTarget.EncryptedConfig("snowplow.databricks.password"))),
+    Some(StorageTarget.Databricks.OAuth("client-id", "client-secret")),
     None,
     "snowplow-rdbloader-oss",
     StorageTarget.LoadAuthMethod.NoCreds,

diff --git a/...ader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala b/...ader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala
@@ -227,7 +227,8 @@ object DatabricksSpec {
       "snowplow",
       443,
       "some/path",
-      StorageTarget.PasswordConfig.PlainText("xxx"),
+      Some(StorageTarget.PasswordConfig.PlainText("xxx")),
+      None,
       None,
       "useragent",
       StorageTarget.LoadAuthMethod.NoCreds,

diff --git a/...loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala b/...loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala
@@ -90,7 +90,7 @@ class CliConfigSpec extends Specification {
     val result = CliConfig.parse[IO](cli).value.unsafeRunSync()
 
     result must beRight.like { case CliConfig(config, _, resolverConfig) =>
-      config.storage.password.getUnencrypted must beEqualTo("Supersecret password from substitution!")
+      config.storage.credentials.get.password.getUnencrypted must beEqualTo("Supersecret password from substitution!")
       resolverConfig must beEqualTo(expectedResolver)
     }
 

diff --git a/...er/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/StorageTargetSpec.scala b/...er/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/StorageTargetSpec.scala
@@ -162,6 +162,10 @@ class StorageTargetSpec extends Specification {
       "port": 443,
       "httpPath": "http/path",
       "password": "Supersecret1",
+      "oauth": {
+        "clientId": "client-id",
+        "clientSecret": "client-secret"
+      },
       "userAgent": "snowplow-rdbloader-oss",
       "eventsOptimizePeriod": "2 days",
       "loadAuthMethod": {
@@ -177,7 +181,8 @@ class StorageTargetSpec extends Specification {
         schema               = "snowplow",
         port                 = 443,
         httpPath             = "http/path",
-        password             = StorageTarget.PasswordConfig.PlainText("Supersecret1"),
+        password             = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")),
+        oauth                = Some(StorageTarget.Databricks.OAuth("client-id", "client-secret")),
         sshTunnel            = None,
         userAgent            = "snowplow-rdbloader-oss",
         loadAuthMethod       = StorageTarget.LoadAuthMethod.NoCreds,

diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala
@@ -243,7 +243,13 @@ object BuildSettings {
       Docker / packageName := "rdb-loader-databricks",
       initialCommands := "import com.snowplowanalytics.snowplow.loader.databricks._",
       Compile / mainClass := Some("com.snowplowanalytics.snowplow.loader.databricks.Main"),
-      Compile / unmanagedJars += file("DatabricksJDBC42.jar")
+      Compile / unmanagedJars += file("DatabricksJDBC42.jar"),
+      // used in extended configuration parsing unit tests
+      Test / envVars := Map(
+        "OAUTH_CLIENT_SECRET" -> "client-secret"
+      ),
+      // envVars works only when fork is enabled
+      Test / fork := true
     ) ++ buildSettings ++ addExampleConfToTestCp ++ assemblySettings ++ dynVerSettings
 
   lazy val transformerBatchBuildSettings =