in daffodil-cli/src/main/scala/org/apache/daffodil/cli/Main.scala [1187:1950]
def runIgnoreExceptions(arguments: Array[String]): ExitCode.Value = {
val conf = new CLIConf(arguments, STDOUT, STDERR)
// Set the log level to whatever was parsed by the options
setLogLevel(conf.verbose())
val ret: ExitCode.Value = conf.subcommand match {
case Some(conf.parse) => {
val parseOpts = conf.parse
val validate = parseOpts.validate.toOption.get
val optDafConfig = parseOpts.config.toOption.map { DaffodilConfig.fromFile(_) }
val processor: Option[DFDL.DataProcessor] = {
if (parseOpts.parser.isDefined) {
createProcessorFromParser(parseOpts.parser(), parseOpts.path.toOption, validate)
} else {
val tunables =
DaffodilTunables.configPlusMoreTunablesMap(parseOpts.tunables, optDafConfig)
createProcessorFromSchema(
parseOpts.schema(),
parseOpts.rootNS.toOption,
parseOpts.path.toOption,
tunables,
validate
)
}
}.map {
_.withExternalVariables(combineExternalVariables(parseOpts.vars, optDafConfig))
}.map { _.withValidationMode(validate) }
.map { withDebugOrTrace(_, parseOpts.trace, parseOpts.debug) }
val rc = processor match {
case None => ExitCode.UnableToCreateProcessor
case Some(processor) => {
Assert.invariant(!processor.isError)
val input = parseOpts.infile.toOption match {
case Some("-") | None => InputSourceDataInputStream(STDIN)
case Some(file) => {
// for files <= 2GB, use a mapped byte buffer to avoid the overhead related to
// the BucketingInputSource. Larger files cannot be mapped so we cannot avoid it
val path = Paths.get(file)
val size = Files.size(path)
if (size <= Int.MaxValue) {
val fc = FileChannel.open(path, StandardOpenOption.READ)
val bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, size)
fc.close() // we no longer need the channel now that we've mapped it
InputSourceDataInputStream(bb)
} else {
val is = Files.newInputStream(path, StandardOpenOption.READ)
InputSourceDataInputStream(is)
}
}
}
Using.resource(input) { inStream =>
val output = parseOpts.output.toOption match {
case Some("-") | None => STDOUT
case Some(file) => new FileOutputStream(file)
}
val infosetHandler = InfosetType.getInfosetHandler(
parseOpts.infosetType(),
processor,
parseOpts.schema.map(_.uri).toOption,
forPerformance = false
)
var lastParseBitPosition = 0L
var keepParsing = true
var exitCode = ExitCode.Success
while (keepParsing) {
val infosetResult =
Timer.getResult("parsing", infosetHandler.parse(inStream, output))
val parseResult = infosetResult.parseResult
val finfo = parseResult.resultState.asInstanceOf[FormatInfo]
val loc = parseResult.resultState.currentLocation.asInstanceOf[DataLoc]
displayDiagnostics(parseResult)
if (parseResult.isProcessingError || parseResult.isValidationError) {
keepParsing = false
exitCode = ExitCode.ParseError
} else {
// Success. Some InfosetHandlers do not write the result to the output
// stream when parsing (e.g. they just create Scala objects). Since we are
// parsing, ask the InfosetHandler to serialize the result if it has an
// implementation so that the user can see an XML represenation regardless
// of the infoset type.
infosetResult.write(output)
output.flush()
if (!inStream.hasData()) {
// not even 1 more bit is available.
// do not try to keep parsing, nothing left to parse
keepParsing = false
} else {
// There is more data available.
if (parseOpts.stream.toOption.get) {
// Streaming mode
if (lastParseBitPosition == loc.bitPos0b) {
// this parse consumed no data, that means this would get
// stuck in an infinite loop if we kept trying to stream,
// so we need to quit
val remainingBits =
if (loc.bitLimit0b.isDefined) {
(loc.bitLimit0b.get - loc.bitPos0b).toString
} else {
"at least " + (inStream.inputSource.knownBytesAvailable() * 8)
}
Logger.log.error(
s"Left over data after consuming 0 bits while streaming. Stopped after consuming ${loc.bitPos0b} bit(s) with ${remainingBits} bit(s) remaining."
)
keepParsing = false
exitCode = ExitCode.LeftOverData
} else {
// last parse did consume data, and we know there is more
// data to come, so try to parse again.
lastParseBitPosition = loc.bitPos0b
keepParsing = true
output.write(0) // NUL-byte separates streams
}
} else {
// not streaming mode, and there is more data available,
// so show left over data warning
val Dump = new DataDumper
val bitsAlreadyConsumed = (loc.bitPos0b % 8).toInt
val firstByteString = if (bitsAlreadyConsumed != 0) {
val bitsToDisplay = 8 - bitsAlreadyConsumed
val pbp = inStream.inputSource.position() + 1
val firstByteBitArray = inStream.getByteArray(bitsToDisplay, finfo)
val fbs = firstByteBitArray(0).toInt.toBinaryString
.takeRight(8)
.reverse
.padTo(8, '0')
.reverse
val bits = if (finfo.bitOrder == BitOrder.MostSignificantBitFirst) {
"x" * bitsAlreadyConsumed + fbs.dropRight(bitsAlreadyConsumed)
} else {
fbs.takeRight(bitsToDisplay) + "x" * bitsAlreadyConsumed
}
val dumpString =
f"\nLeft over data starts with partial byte. Left over data (Binary) at byte $pbp is: (0b$bits)"
dumpString
} else ""
val curBytePosition1b = inStream.inputSource.position() + 1
val bytesAvailable = inStream.inputSource.knownBytesAvailable()
val bytesLimit = math.min(8, bytesAvailable).toInt
val destArray = new Array[Byte](bytesLimit)
val destArrayFilled = inStream.inputSource.get(destArray, 0, bytesLimit)
val dumpString =
if (destArrayFilled)
Dump
.dump(
Dump.TextOnly(Some("utf-8")),
0,
destArray.length * 8,
ByteBuffer.wrap(destArray),
includeHeadingLine = false
)
.mkString("\n")
else ""
val dataText =
if (destArrayFilled)
s"\nLeft over data (UTF-8) starting at byte ${curBytePosition1b} is: (${dumpString}...)"
else ""
val dataHex =
if (destArrayFilled)
s"\nLeft over data (Hex) starting at byte ${curBytePosition1b} is: (0x${destArray.map { a =>
f"$a%02x"
}.mkString}...)"
else ""
val remainingBits =
if (loc.bitLimit0b.isDefined) {
(loc.bitLimit0b.get - loc.bitPos0b).toString
} else {
"at least " + (bytesAvailable * 8)
}
val leftOverDataMessage =
s"Left over data. Consumed ${loc.bitPos0b} bit(s) with ${remainingBits} bit(s) remaining." + firstByteString + dataHex + dataText
Logger.log.error(leftOverDataMessage)
keepParsing = false
exitCode = ExitCode.LeftOverData
}
}
}
}
exitCode
}
}
}
rc
}
case Some(conf.performance) => {
val performanceOpts = conf.performance
val validate = performanceOpts.validate.toOption.get
val optDafConfig = performanceOpts.config.toOption.map { DaffodilConfig.fromFile(_) }
val processor = {
if (performanceOpts.parser.isDefined) {
createProcessorFromParser(
performanceOpts.parser(),
performanceOpts.path.toOption,
validate
)
} else {
val tunables =
DaffodilTunables.configPlusMoreTunablesMap(performanceOpts.tunables, optDafConfig)
createProcessorFromSchema(
performanceOpts.schema(),
performanceOpts.rootNS.toOption,
performanceOpts.path.toOption,
tunables,
validate
)
}
}.map {
_.withExternalVariables(combineExternalVariables(performanceOpts.vars, optDafConfig))
}.map { _.withValidationMode(validate) }
val rc: ExitCode.Value = processor match {
case None => ExitCode.UnableToCreateProcessor
case Some(processor) => {
val infile = new java.io.File(performanceOpts.infile())
val files = {
if (infile.isDirectory()) {
infile.listFiles.filter(!_.isDirectory)
} else {
Array(infile)
}
}
val infosetHandler = InfosetType.getInfosetHandler(
performanceOpts.infosetType(),
processor,
performanceOpts.schema.map(_.uri).toOption,
forPerformance = true
)
val dataSeq: Seq[Either[AnyRef, Array[Byte]]] =
ArraySeq
.unsafeWrapArray(files)
.map { filePath =>
// For performance testing, we want everything in memory so as to
// remove I/O from consideration. Additionally, for both parse
// and unparse we need immutable inputs since we could parse the
// same input data multiple times in different performance runs.
// So read the file data into an Array[Byte], and use that for
// everything.
val input = (new FileInputStream(filePath))
val dataSize = filePath.length()
val bytes = new Array[Byte](dataSize.toInt)
input.read(bytes)
val data = performanceOpts.unparse() match {
case true => Left(infosetHandler.dataToInfoset(bytes))
case false => Right(bytes)
}
data
}
val inputs = (0 until performanceOpts.number()).map { n =>
val index = n % dataSeq.length
dataSeq(index)
}
val inputsWithIndex = inputs.zipWithIndex
implicit val executionContext = new ExecutionContext {
val threadPool = Executors.newFixedThreadPool(performanceOpts.threads())
def execute(runnable: Runnable): Unit = {
threadPool.submit(runnable)
}
def reportFailure(t: Throwable): Unit = {
// do nothing
}
}
val nullChannelForUnparse = Channels.newChannel(NullOutputStream.INSTANCE)
val nullOutputStreamForParse = NullOutputStream.INSTANCE
val NSConvert = 1000000000.0
val (totalTime, results) = Timer.getTimeResult({
val tasks = inputsWithIndex.map { case (inData, n) =>
val task: Future[(Int, Long, Boolean)] = Future {
val (time, result) = inData match {
case Left(anyRef) =>
Timer.getTimeResult({
val unparseResult =
infosetHandler.unparse(anyRef, nullChannelForUnparse)
unparseResult
})
case Right(bytes) =>
Timer.getTimeResult({
Using.resource(InputSourceDataInputStream(bytes)) { input =>
val infosetResult =
infosetHandler.parse(input, nullOutputStreamForParse)
val parseResult = infosetResult.parseResult
parseResult
}
})
}
(n, time, result.isError)
}
task
}
val results = tasks.map { Await.result(_, Duration.Inf) }
results
})
val rates = results.map { results =>
val (runNum: Int, nsTime: Long, error: Boolean) = results
val rate = 1 / (nsTime / NSConvert)
val status = if (error) "fail" else "pass"
Logger.log.info(
s"run: ${runNum}, seconds: ${nsTime / NSConvert}, rate: ${rate}, status: ${status}"
)
rate
}
val numFailures = results.map { _._3 }.filter { e => e }.length
if (numFailures > 0) {
Logger.log.error(s"${numFailures} failures found\n")
}
val sec = totalTime / NSConvert
val action = performanceOpts.unparse() match {
case true => "unparse"
case false => "parse"
}
STDOUT.println(s"total $action time (sec): $sec")
STDOUT.println(s"min rate (files/sec): ${rates.min.toFloat}")
STDOUT.println(s"max rate (files/sec): ${rates.max.toFloat}")
STDOUT.println(s"avg rate (files/sec): ${(performanceOpts.number() / sec).toFloat}")
if (numFailures == 0) ExitCode.Success else ExitCode.PerformanceTestError
}
}
rc
}
case Some(conf.unparse) => {
val unparseOpts = conf.unparse
val validate = unparseOpts.validate.toOption.get
val optDafConfig = unparseOpts.config.toOption.map { DaffodilConfig.fromFile(_) }
val processor = {
if (unparseOpts.parser.isDefined) {
createProcessorFromParser(unparseOpts.parser(), unparseOpts.path.toOption, validate)
} else {
val tunables =
DaffodilTunables.configPlusMoreTunablesMap(unparseOpts.tunables, optDafConfig)
createProcessorFromSchema(
unparseOpts.schema(),
unparseOpts.rootNS.toOption,
unparseOpts.path.toOption,
tunables,
validate
)
}
}.map {
_.withExternalVariables(combineExternalVariables(unparseOpts.vars, optDafConfig))
}.map { _.withValidationMode(validate) }
.map { withDebugOrTrace(_, unparseOpts.trace, unparseOpts.debug) }
val output = unparseOpts.output.toOption match {
case Some("-") | None => STDOUT
case Some(file) => new FileOutputStream(file)
}
val outChannel = Channels.newChannel(output)
//
// We are not loading a schema here, we're loading the infoset to unparse.
//
val is = unparseOpts.infile.toOption match {
case Some("-") | None => STDIN
case Some(fileName) => new FileInputStream(fileName)
}
val rc = processor match {
case None => ExitCode.UnableToCreateProcessor
case Some(processor) => {
Assert.invariant(processor.isError == false)
val maybeScanner =
if (unparseOpts.stream.toOption.get) {
val scnr = new Scanner(is)
scnr.useDelimiter("\u0000")
Some(scnr)
} else {
None
}
var keepUnparsing = maybeScanner.isEmpty || maybeScanner.get.hasNext
var exitCode = ExitCode.Success
val infosetHandler = InfosetType.getInfosetHandler(
unparseOpts.infosetType(),
processor,
unparseOpts.schema.map(_.uri).toOption,
forPerformance = false
)
while (keepUnparsing) {
val inputterData =
if (maybeScanner.isDefined) {
// The scanner reads the entire infoset up unto the delimiter
// into memory. No way around that with the --stream option.
val bytes = maybeScanner.get.next().getBytes()
infosetHandler.dataToInfoset(bytes)
} else {
// We are not using the --stream option and won't need to
// unparse the infoset more than once. So pass the
// InputStream into dataToInfoset. For some cases, such as
// "xml" or "json", we can create an InfosetInputter directly
// on this stream so that we can avoid reading the entire
// InputStream into memory
infosetHandler.dataToInfoset(is)
}
val unparseResult =
Timer.getResult("unparsing", infosetHandler.unparse(inputterData, outChannel))
displayDiagnostics(unparseResult)
if (unparseResult.isError) {
keepUnparsing = false
exitCode = ExitCode.UnparseError
} else {
keepUnparsing = maybeScanner.isDefined && maybeScanner.get.hasNext
}
}
exitCode
}
}
is.close()
outChannel.close()
rc
}
case Some(conf.save) => {
val saveOpts = conf.save
val validate = ValidationMode.Off
val optDafConfig = saveOpts.config.toOption.map { DaffodilConfig.fromFile(_) }
val tunables =
DaffodilTunables.configPlusMoreTunablesMap(saveOpts.tunables, optDafConfig)
val tunablesObj = DaffodilTunables(tunables)
val processor = createProcessorFromSchema(
saveOpts.schema(),
saveOpts.rootNS.toOption,
saveOpts.path.toOption,
tunables,
validate
)
val output = saveOpts.outfile.toOption match {
case Some("-") | None => Channels.newChannel(STDOUT)
case Some(file) => new FileOutputStream(file).getChannel()
}
val rc = processor match {
case Some(processor) => {
Assert.invariant(processor.isError == false)
Timer.getResult("saving", processor.save(output))
ExitCode.Success
}
case None => ExitCode.UnableToCreateProcessor
}
rc
}
case Some(conf.test) => {
val testOpts = conf.test
val tdmlFile = testOpts.tdmlfile()
val optTDMLImplementation = testOpts.implementation.toOption
val tdmlRunnerInit = Runner(tdmlFile, optTDMLImplementation)
val tdmlRunner = if (testOpts.trace() || testOpts.debug.isDefined) {
val db = getTraceOrCLIDebuggerRunner(testOpts.trace, testOpts.debug)
val id = new InteractiveDebugger(db, ExpressionCompilers)
tdmlRunnerInit.setDebugger(id)
tdmlRunnerInit
} else {
tdmlRunnerInit
}
val tests = {
if (testOpts.testnames.isDefined) {
testOpts
.testnames()
.flatMap(testName => {
if (testOpts.regex()) {
val regex = testName.r
val matches = tdmlRunner
.testCases()
.filter(testCase => regex.pattern.matcher(testCase.tcName).matches)
matches match {
case m if !m.isEmpty => m.map(testCase => (testCase.tcName, Some(testCase)))
case _ => Seq((testName, None))
}
} else {
List((testName, tdmlRunner.testCases().find(_.tcName == testName)))
}
})
} else {
tdmlRunner.testCases().map(test => (test.tcName, Some(test)))
}
}.distinct.sortBy(_._1)
tdmlRunner.reset()
if (testOpts.list()) {
if (testOpts.info() > 0) {
// determine the max lengths of the various pieces of a test
val headers = List("Name", "Model", "Root", "Description")
val maxCols = tests.foldLeft(headers.map(_.length)) { (maxVals, testPair) =>
{
testPair match {
case (name, None) =>
List(maxVals(0).max(name.length), maxVals(1), maxVals(2), maxVals(3))
case (name, Some(test)) =>
List(
maxVals(0).max(name.length),
maxVals(1).max(test.model.length),
maxVals(2).max(test.rootName.length),
maxVals(3).max(test.description.length)
)
}
}
}
val formatStr = maxCols.map(max => "%" + -max + "s").mkString(" ")
STDOUT.println(formatStr.format(headers: _*))
tests.foreach { testPair =>
testPair match {
case (name, Some(test)) =>
STDOUT.println(
formatStr.format(name, test.model, test.rootName, test.description)
)
case (name, None) =>
STDOUT.println(formatStr.format(name, "[Not Found]", "", ""))
}
}
} else {
tests.foreach { testPair =>
testPair match {
case (name, Some(test)) => STDOUT.println(name)
case (name, None) => STDOUT.println("%s [Not Found]".format(name))
}
}
}
ExitCode.Success
} else {
var pass = 0
var fail = 0
var notfound = 0
tests.foreach { testPair =>
testPair match {
case (name, Some(test)) => {
try {
test.run()
STDOUT.println("[Pass] %s".format(name))
pass += 1
} catch {
case s: scala.util.control.ControlThrowable => throw s
case u: UnsuppressableException => throw u
case e: TDMLTestNotCompatibleException => {
STDOUT.println(
"[Skipped] %s (not compatible with implementation: %s)"
.format(name, e.implementation.getOrElse("<none>"))
)
}
case e: Throwable => {
e.getCause match {
case e: TDMLTestNotCompatibleException => {
// if JUnit is on the classpath (e.g. possible in integration tests), then
// the TDML runner throws an AssumptionViolatedException where the cause is
// a TDMLTestNotCompatibleException. In that case we should output the test
// skipped message. The way we match here avoids having the CLI to require
// JUnit as a dependency
STDOUT.println(
"[Skipped] %s (not compatible with implementation: %s)"
.format(name, e.implementation.getOrElse("<none>"))
)
}
case _ => {
STDOUT.println("[Fail] %s".format(name))
fail += 1
if (testOpts.info() > 0) {
STDOUT.println(" Failure Information:")
STDOUT.println(indent(e.toString, 4))
}
if (testOpts.info() > 1) {
STDOUT.println(" Backtrace:")
e.getStackTrace.foreach { st =>
STDOUT.println(indent(st.toString, 4))
}
}
}
}
}
}
}
case (name, None) => {
STDOUT.println("[Not Found] %s".format(name))
notfound += 1
}
}
}
STDOUT.println("")
STDOUT.println(
"Total: %d, Pass: %d, Fail: %d, Not Found: %s".format(
pass + fail + notfound,
pass,
fail,
notfound
)
)
if (fail == 0 && notfound == 0) ExitCode.Success else ExitCode.TestError
}
}
// Get our generate options from whichever language we're generating
case Some(conf.generate) => {
// Determine which language we've generating
val generateOpts = conf.generate.subcommand match {
case Some(conf.generate.c) => conf.generate.c
// Required to avoid "match may not be exhaustive", but should never happen
case _ => Assert.impossible()
}
// Read any config file and any tunables given as arguments
val optDafConfig = generateOpts.config.toOption.map { DaffodilConfig.fromFile(_) }
val tunables =
DaffodilTunables.configPlusMoreTunablesMap(generateOpts.tunables, optDafConfig)
// Create a CodeGenerator from the DFDL schema
val generator = createGeneratorFromSchema(
generateOpts.schema(),
generateOpts.rootNS.toOption,
tunables,
generateOpts.language
)
// Ask the CodeGenerator to generate source code from the DFDL schema
val outputDir = generateOpts.outdir.toOption.getOrElse(".")
val rc = generator match {
case Some(generator) => {
Timer.getResult("generating", generator.generateCode(outputDir))
displayDiagnostics(generator)
if (generator.isError) ExitCode.GenerateCodeError else ExitCode.Success
}
case None => ExitCode.GenerateCodeError
}
rc
}
case Some(conf.exi) => {
var rc = ExitCode.Success
val exiOpts = conf.exi
val channel = exiOpts.output.toOption match {
case Some("-") | None => Channels.newChannel(STDOUT)
case Some(file) => new FileOutputStream(file).getChannel()
}
val output = Channels.newOutputStream(channel)
val inputStream = exiOpts.infile.toOption match {
case Some("-") | None => STDIN
case Some(file) => {
val f = new File(file)
new FileInputStream(f)
}
}
val input = new InputSource(inputStream)
val exiFactory: Option[EXIFactory] =
try {
Some(
EXIInfosetHandler.createEXIFactory(
exiOpts.schema.map(_.uri).toOption
)
)
} catch {
case e: EXIException => {
Logger.log.error(
s"Error creating EXI grammar for the supplied schema: ${Misc.getSomeMessage(e).get}"
)
rc = ExitCode.Failure
None
}
}
(exiOpts.decode.toOption.get, exiFactory.isDefined) match {
case (true, true) => { // Decoding
val exiSource = new EXISource(exiFactory.get)
exiSource.setInputSource(input)
val result = new StreamResult(output)
val tf = TransformerFactory.newInstance()
val transformer = tf.newTransformer
transformer.setErrorListener(new EXIErrorHandler)
try {
transformer.transform(exiSource, result)
} catch {
/* We catch a generic Exception here as Exificient will attempt
* to decode anything and will throw very generic errors, such as
* an IllegalArgumentException when it runs into a series of bytes
* that aren't a Unicode codepoint. This should be removed once
* https://github.com/EXIficient/exificient/issues/33 is fixed.*/
case e: Exception => {
Logger.log.error(s"Error decoding EXI input: ${Misc.getSomeMessage(e).get}")
rc = ExitCode.Failure
}
}
}
case (false, true) => { // Encoding
val exiResult = new EXIResult(exiFactory.get)
exiResult.setOutputStream(output)
val factory = SAXParserFactory.newInstance()
factory.setNamespaceAware(true)
val saxParser = factory.newSAXParser()
val reader = saxParser.getXMLReader
reader.setContentHandler(exiResult.getHandler)
reader.setErrorHandler(new EXIErrorHandler)
try {
reader.parse(input)
} catch {
case s: org.xml.sax.SAXException => {
Logger.log.error(s"Error parsing input XML: ${Misc.getSomeMessage(s).get}")
rc = ExitCode.Failure
}
}
}
case (_, false) => // Hit an exception creating exiFactory, rc already set
}
inputStream.close
output.close
rc
}
// Required to avoid "match may not be exhaustive", but should never happen
case _ => Assert.impossible()
}
ret
}