in server/src/main/scala/org/apache/livy/sessions/Session.scala [54:102]
def prepareConf(
conf: Map[String, String],
jars: Seq[String],
files: Seq[String],
archives: Seq[String],
pyFiles: Seq[String],
livyConf: LivyConf): Map[String, String] = {
if (conf == null) {
return Map()
}
val errors = if (livyConf.getBoolean(LivyConf.SESSION_ALLOW_CUSTOM_CLASSPATH)) {
conf.keySet.filter(configBlackList.contains)
} else {
conf.keySet.filter((configBlackList ++ BLACKLIST_CUSTOM_CLASSPATH).contains)
}
if (errors.nonEmpty) {
throw new IllegalArgumentException(
"Blacklisted configuration values in session config: " + errors.mkString(", "))
}
val confLists: Map[String, Seq[String]] = livyConf.sparkFileLists
.map { key => (key -> Nil) }.toMap
val userLists = confLists ++ Map(
LivyConf.SPARK_JARS -> jars,
LivyConf.SPARK_FILES -> files,
LivyConf.SPARK_ARCHIVES -> archives,
LivyConf.SPARK_PY_FILES -> pyFiles)
val merged = userLists.flatMap { case (key, list) =>
val confList = conf.get(key)
.map { list =>
resolveURIs(list.split("[, ]+").toSeq, livyConf)
}
.getOrElse(Nil)
val userList = resolveURIs(list, livyConf)
if (confList.nonEmpty || userList.nonEmpty) {
Some(key -> (userList ++ confList).mkString(","))
} else {
None
}
}
val masterConfList = Map(LivyConf.SPARK_MASTER -> livyConf.sparkMaster()) ++
livyConf.sparkDeployMode().map(LivyConf.SPARK_DEPLOY_MODE -> _).toMap
conf ++ masterConfList ++ merged
}