in scio-core/src/main/scala/com/spotify/scio/ScioContext.scala [323:383]
def parseArguments[T <: PipelineOptions: ClassTag](
cmdlineArgs: Array[String],
withValidation: Boolean = false
): (T, Args) = {
val optClass = ScioUtil.classOf[T]
PipelineOptionsFactory.register(optClass)
// Extract --pattern of all registered derived types of PipelineOptions
val registeredPatterns = for {
cls <- PipelineOptionsFactory.getRegisteredOptions.asScala
method <- cls.getMethods()
name = method.getName
str <-
if (
(!name.startsWith("get") && !name.startsWith("is")) ||
method.getParameterTypes.nonEmpty || method.getReturnType == classOf[Unit]
) {
None
} else {
Some(Introspector.decapitalize(name.substring(if (name.startsWith("is")) 2 else 3)))
}
} yield s"--$str($$|=)".r
val patterns = registeredPatterns.toSet + "--help($$|=)".r
// Split cmdlineArgs into 2 parts, optArgs for PipelineOptions and appArgs for Args
val (optArgs, appArgs) =
cmdlineArgs.partition(arg => patterns.exists(_.findFirstIn(arg).isDefined))
val pipelineOpts = if (withValidation) {
PipelineOptionsFactory.fromArgs(optArgs: _*).withValidation().as(optClass)
} else {
PipelineOptionsFactory.fromArgs(optArgs: _*).as(optClass)
}
val optionsFile = pipelineOpts.as(classOf[ScioOptions]).getOptionsFile
if (optionsFile != null) {
log.info(s"Appending options from $optionsFile")
parseArguments(
cmdlineArgs.filterNot(_.startsWith("--optionsFile=")) ++
Source.fromFile(optionsFile).getLines()
)
} else {
val args = Args(appArgs)
if (appArgs.nonEmpty) {
val argString = args.toString("", ", ", "")
val sanitizedArgString =
if (argString.length > appArgStringMaxLength) {
log.warn("Truncating long app arguments")
argString.substring(0, appArgStringMaxLength) + " [...]"
} else {
argString
}
pipelineOpts
.as(classOf[ScioOptions])
.setAppArguments(sanitizedArgString)
}
(pipelineOpts, args)
}
}