如何在Spark jar中动态替换typesafe include语句

Question

此方法替换了资源目录中放置的文件中的typsafe include文件路径。它在本地工作但在GCP数据交换机上的jar中运行时失败

def getTypeSafeConfig(conf: DatalakeConfig): Config = {

 val env = conf.env

 val resourcesPath = getClass
  .getResource("/activity/mlmActivityProducer.conf")
  .getPath.replace("/activity/mlmActivityProducer.conf", "")

 println(resourcesPath)

 val accountsConfigPath = s"${resourcesPath}/accounts/$env/accounts.conf"
 val mlmConfigPath = s"${resourcesPath}/mlm.conf"
 val accountsTestScenarioConfigPath = s"${resourcesPath}/activity/testScenario.conf"

val stream = 
 getClass.getResourceAsStream("/activity/mlmActivityProducer.conf")
 val lines = scala.io.Source.fromInputStream( stream ).getLines

 var s = ""

 lines.foreach(l => {

  val ll = l
    .replace(ACCOUNTS_CONFIG_PATH, accountsConfigPath)
    .replace(MLM_CONFIG_PATH, mlmConfigPath)
    .replace(ACCOUNTS_TEST_SCENARIO_CONFIG_PATH, 
   accountsTestScenarioConfigPath)

  //      println(s"floki: ${ll}")
  s += s"\n${ll}"

 })

 println(s"res: $s")



 ConfigFactory.parseString(s)
}

这些是在Dataproc上分配的路径：

include file("file:/tmp/87a59d76c5814f30aa29935f397593cf/common-1.0.0-SNAPSHOT.jar!/accounts/qe/accounts.conf")
include file("file:/tmp/87a59d76c5814f30aa29935f397593cf/common-1.0.0-SNAPSHOT.jar!/mlm.conf")
include file("file:/tmp/87a59d76c5814f30aa29935f397593cf/common-1.0.0-SNAPSHOT.jar!/activity/testScenario.conf")

Answer 1

我用错误的方式更好地使用后备

def getTypeSafeConfig(conf: DatalakeConfig): Config = {

  val env = conf.env

  val baseConfig = ConfigFactory.parseResources("activity/mlmActivityProducer.conf")
  val envAccountConfig = ConfigFactory.parseResources(s"accounts/$env/accounts.conf")
  val mlmConfig = ConfigFactory.parseResources("mlm.conf")
  val testScenarioConfig = ConfigFactory.parseResources(s"activity/testScenario.conf")
  val typeSafeConfig = testScenarioConfig
    .withFallback(envAccountConfig)
    .withFallback(mlmConfig)
    .withFallback(baseConfig)
    .resolve()

   println(typeSafeConfig.toString)

   typeSafeConfig
}