in datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala [82:155]
def getAbsolutePaths(): Seq[String] = resources.map(_.resolvedLocation).distinct
def getAbsolutePathsForJava(): util.List[String] =
resources.map(_.resolvedLocation).distinct.asJava
def getPYTHONPATH(): String =
resources
.map(_.resolvedLocation)
.map(p => new File(p))
.map(_.getName) // get just the name of the file
.mkString(":")
private def resolveDependencyLocation(resource: PythonResource): String =
if (resource.isAbsolutePath) {
if (!new File(resource.resourcePath).exists()) {
throw new IOException(
"Could not find resource in absolute path: " + resource.resourcePath)
} else {
logger.info("Using file absolute path: " + resource.resourcePath)
resource.resourcePath
}
} else {
Option(getClass.getClassLoader.getResource(resource.resourcePath)) match {
case None =>
logger.error(
"Didn't find resource in classpath! resource path: " + resource.resourcePath)
throw new MissingResourceException(
"Didn't find resource in classpath!",
resource.getClass.getName,
resource.resourcePath)
case Some(p) =>
p.toURI.getScheme match {
case "jar" =>
// if dependency is inside jar file, use jar file path:
val jarPath = new File(
p.openConnection()
.asInstanceOf[JarURLConnection]
.getJarFileURL
.toURI).getPath
logger.info(
s"Dependency ${resource.resourcePath} found inside jar: " + jarPath)
jarPath
case "file" =>
val file = new File(p.getFile)
if (!file.exists()) {
logger.warn("Dependency not found, skipping: " + file.getPath)
null
} else {
if (file.isDirectory) {
val t_path =
if (System
.getProperty("os.name")
.toLowerCase()
.contains("win") && p.getPath().startsWith("/")) {
val path = p.getPath.substring(1)
logger.warn(
s"Fixing path for windows operating system! " +
s"converted ${p.getPath} to $path")
path
} else {
p.getPath
}
val path = Paths.get(t_path)
logger.info(
s"Dependency found as directory: ${t_path}\n\tusing " +
s"parent path: ${path.getParent}")
path.getParent.toString
} else {
logger.info("Dependency found as a file: " + p.getPath)
p.getPath
}
}
}
}
}