build.sbt (519 lines of code) (raw):

/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import scala.collection.immutable.ListSet import sbtcc._ lazy val genManaged = taskKey[Unit]("Generate managed sources and resources") lazy val genProps = taskKey[Seq[File]]("Generate properties scala source") lazy val genSchemas = taskKey[Seq[File]]("Generate DFDL schemas") lazy val genCExamples = taskKey[Seq[File]]("Generate C example files") lazy val genVersion = taskKey[Seq[File]]("Generate VERSION file") lazy val genTunablesDoc = taskKey[Seq[File]]("Generate tunables doc from dafext.xsd file") lazy val daffodil = project .in(file(".")) .enablePlugins(JavaUnidocPlugin, ScalaUnidocPlugin) .aggregate( cli, codeGenC, core, io, japi, lib, macroLib, propgen, runtime1, runtime1Layers, runtime1Unparser, sapi, schematron, slf4jLogger, tdmlJunit, tdmlLib, tdmlProc, testDaf, testIBM1, // testIntegration, // integration tests must be run manually testStdLayout, tutorials, udf ) .settings( commonSettings, nopublish, ratSettings, unidocSettings, genTunablesDocSettings, genCExamplesSettings ) lazy val macroLib = Project("daffodil-macro-lib", file("daffodil-macro-lib")) .settings(commonSettings, nopublish) .settings(libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value) .disablePlugins(OsgiCheckPlugin) lazy val propgen = Project("daffodil-propgen", file("daffodil-propgen")) .settings(commonSettings, nopublish) lazy val slf4jLogger = Project("daffodil-slf4j-logger", file("daffodil-slf4j-logger")) .settings(commonSettings) .settings(libraryDependencies ++= Dependencies.slf4jAPI) lazy val lib = Project("daffodil-lib", file("daffodil-lib")) .dependsOn(macroLib % "compile-internal, test-internal", slf4jLogger % "test") .settings(commonSettings, libManagedSettings, usesMacros) lazy val io = Project("daffodil-io", file("daffodil-io")) .dependsOn(lib, macroLib % "compile-internal, test-internal", slf4jLogger % "test") .settings(commonSettings, usesMacros) lazy val runtime1 = Project("daffodil-runtime1", file("daffodil-runtime1")) .enablePlugins(GenJavadocPlugin) .settings(Dependencies.genjavadocVersion) // converts scaladoc to javadoc .dependsOn( io, lib % "compile-internal, test->test", udf, macroLib % "compile-internal, test-internal", slf4jLogger % "test" ) .settings(commonSettings, usesMacros) lazy val runtime1Unparser = Project("daffodil-runtime1-unparser", file("daffodil-runtime1-unparser")) .dependsOn( runtime1, lib % "test->test", runtime1 % "test->test", runtime1Layers, slf4jLogger % "test" ) .settings(commonSettings) lazy val runtime1Layers = Project("daffodil-runtime1-layers", file("daffodil-runtime1-layers")) .dependsOn(runtime1, lib % "test->test", slf4jLogger % "test") .settings(commonSettings) val codeGenCLib = Library("libruntime.a") lazy val codeGenC = Project("daffodil-codegen-c", file("daffodil-codegen-c")) .enablePlugins(CcPlugin) .dependsOn(core, core % "test->test", slf4jLogger % "test") .settings(commonSettings) .settings( Compile / cCompiler := sys.env.getOrElse("CC", "cc"), Compile / ccArchiveCommand := sys.env.getOrElse("AR", "ar"), Compile / ccTargets := ListSet(codeGenCLib), Compile / cSources := Map( codeGenCLib -> ((Compile / resourceDirectory).value / "org" / "apache" / "daffodil" / "codegen" / "c" / "files" * GlobFilter("lib*") * GlobFilter("*.c")).get() ), Compile / cIncludeDirectories := Map( codeGenCLib -> Seq( (Compile / resourceDirectory).value / "org" / "apache" / "daffodil" / "codegen" / "c" / "files" / "libcli", (Compile / resourceDirectory).value / "org" / "apache" / "daffodil" / "codegen" / "c" / "files" / "libruntime" ) ), Compile / cFlags := (Compile / cFlags).value .withDefaultValue(Seq("-Wall", "-Wextra", "-Wpedantic", "-std=gnu11")) ) lazy val core = Project("daffodil-core", file("daffodil-core")) .dependsOn( runtime1Unparser, udf, lib % "test->test", runtime1 % "test->test", io % "test->test", slf4jLogger % "test" ) .settings(commonSettings) lazy val japi = Project("daffodil-japi", file("daffodil-japi")) .enablePlugins(GenJavadocPlugin) .settings(Dependencies.genjavadocVersion) // converts scaladoc to javadoc .dependsOn(core, slf4jLogger % "test") .settings(commonSettings) lazy val sapi = Project("daffodil-sapi", file("daffodil-sapi")) .dependsOn(core, slf4jLogger % "test") .settings(commonSettings) lazy val tdmlLib = Project("daffodil-tdml-lib", file("daffodil-tdml-lib")) .dependsOn(macroLib % "compile-internal", lib, io, io % "test->test", slf4jLogger % "test") .settings(commonSettings) lazy val tdmlProc = Project("daffodil-tdml-processor", file("daffodil-tdml-processor")) .dependsOn(tdmlLib, codeGenC, core, slf4jLogger) .settings(commonSettings) lazy val tdmlJunit = Project("daffodil-tdml-junit", file("daffodil-tdml-junit")) .dependsOn(tdmlProc) .settings(commonSettings) .settings(libraryDependencies ++= Dependencies.junit) lazy val cli = Project("daffodil-cli", file("daffodil-cli")) .dependsOn( tdmlProc, codeGenC, sapi, japi, schematron % Runtime, slf4jLogger ) // causes codegen-c/sapi/japi to be pulled into the helper zip/tar .settings(commonSettings, nopublish) .settings(libraryDependencies ++= Dependencies.cli) .settings(libraryDependencies ++= Dependencies.exi) lazy val udf = Project("daffodil-udf", file("daffodil-udf")) .dependsOn(slf4jLogger % "test") .settings(commonSettings) lazy val schematron = Project("daffodil-schematron", file("daffodil-schematron")) .dependsOn(lib, sapi % Test, slf4jLogger % "test") .settings(commonSettings) .settings(libraryDependencies ++= Dependencies.schematron) lazy val testDaf = Project("daffodil-test", file("daffodil-test")) .dependsOn(tdmlJunit % "test", codeGenC % "test->test", udf % "test->test") .settings(commonSettings, nopublish) // // Uncomment the following line to run these tests // against IBM DFDL using the Cross Tester // //.settings(IBMDFDLCrossTesterPlugin.settings) lazy val testIBM1 = Project("daffodil-test-ibm1", file("daffodil-test-ibm1")) .dependsOn(tdmlJunit % "test") .settings(commonSettings, nopublish) // // Uncomment the following line to run these tests // against IBM DFDL using the Cross Tester // //.settings(IBMDFDLCrossTesterPlugin.settings) lazy val testIntegration = Project("daffodil-test-integration", file("daffodil-test-integration")) .dependsOn(cli % "test->test", udf % "test->test", testDaf % "test->test") .settings(commonSettings, nopublish) .settings( // CLI integration tests fork a new process which requires extra memory, so these should // only be run sequentially. We also need to stage the CLI script if any of the test // tasks are run Test / parallelExecution := false, Test / test := (Test / test).dependsOn(cli / Compile / stage).value, Test / testOnly := (Test / testOnly).dependsOn(cli / Compile / stage).evaluated, Test / testQuick := (Test / testQuick).dependsOn(cli / Compile / stage).evaluated ) lazy val tutorials = Project("daffodil-tutorials", file("tutorials")) .dependsOn(tdmlJunit % "test") .settings(commonSettings, nopublish) lazy val testStdLayout = Project("daffodil-test-stdLayout", file("test-stdLayout")) .dependsOn(tdmlJunit % "test") .settings(commonSettings, nopublish) // Choices here are Java LTS versions, 8, 11, 17, 21,... // However 8 is deprecated as of Java 21, so will be phased out. val minSupportedJavaVersion: String = if (scala.util.Properties.isJavaAtLeast("21")) "11" else "8" lazy val commonSettings = Seq( organization := "org.apache.daffodil", version := IO.read((ThisBuild / baseDirectory).value / "VERSION").trim, scalaVersion := "2.13.16", crossScalaVersions := Seq("2.13.16"), scalacOptions ++= buildScalacOptions(scalaVersion.value), Test / scalacOptions ++= buildTestScalacOptions(scalaVersion.value), Compile / compile / javacOptions ++= buildJavacOptions(), logBuffered := true, transitiveClassifiers := Seq("sources", "javadoc"), retrieveManaged := true, useCoursier := false, // disabled because it breaks retrieveManaged (sbt issue #5078) exportJars := true, Test / exportJars := false, publishMavenStyle := true, Test / publishArtifact := false, ThisBuild / pomIncludeRepository := { _ => false }, scmInfo := Some( ScmInfo( browseUrl = url("https://github.com/apache/daffodil"), connection = "scm:git:https://github.com/apache/daffodil" ) ), licenses := Seq(License.Apache2), homepage := Some(url("https://daffodil.apache.org")), releaseNotesURL := Some(url(s"https://daffodil.apache.org/releases/${version.value}/")), unmanagedBase := baseDirectory.value / "lib" / "jars", sourceManaged := baseDirectory.value / "src_managed", resourceManaged := baseDirectory.value / "resource_managed", libraryDependencies ++= Dependencies.common, testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "--verbosity=1"), Compile / packageDoc / publishArtifact := false ) def buildScalacOptions(scalaVersion: String) = { val commonOptions = Seq( s"-release:$minSupportedJavaVersion", // scala 2.12 can only do Java 8, regardless of this setting. "-feature", "-deprecation", "-language:experimental.macros", "-unchecked", "-Xfatal-warnings", "-Xxml:-coalescing", "-Ywarn-unused:imports" ) val scalaVersionSpecificOptions = CrossVersion.partialVersion(scalaVersion) match { case Some((2, 13)) => Seq( "-Xlint:inaccessible", "-Xlint:infer-any", "-Xlint:nullary-unit", // suppress nullary-unit warning in the specific trait "-Wconf:cat=lint-nullary-unit:silent,site=org.apache.daffodil.junit.tdml.TdmlTests:silent" ) case _ => Seq.empty } commonOptions ++ scalaVersionSpecificOptions } def buildTestScalacOptions(scalaVersion: String) = { val commonOptions = Seq.empty val scalaVersionSpecificOptions = CrossVersion.partialVersion(scalaVersion) match { case Some((2, 12)) => Seq.empty case Some((2, 13)) => Seq( // suppress nullary-unit warning in tests "-Wconf:cat=lint-nullary-unit:silent" ) case _ => Seq.empty } commonOptions ++ scalaVersionSpecificOptions } val javaVersionSpecificOptions = { val releaseOption = // as of Java 11, they no longer accept "-release". Must use "--release". if (scala.util.Properties.isJavaAtLeast("11")) "--release" else "-release" // Java 21 deprecates Java 8 and warns about it. // So if you are using Java 21, Java code compilation will specify a newer Java version // to avoid warnings. if (scala.util.Properties.isJavaAtLeast("11")) Seq(releaseOption, minSupportedJavaVersion) else if (scala.util.Properties.isJavaAtLeast("9")) Seq(releaseOption, "8") else Nil // for Java 8 compilation } // Workaround issue that some options are valid for javac, not javadoc. // These javacOptions are for code compilation only. (Issue sbt/sbt#355) def buildJavacOptions() = { val commonOptions = Seq( "-Werror", "-Xlint:deprecation" ) commonOptions ++ javaVersionSpecificOptions } lazy val nopublish = Seq( publish := {}, publishLocal := {}, publishM2 := {}, publish / skip := true ) // "usesMacros" is a list of settings that should be applied only to // subprojects that use the Daffodil macroLib subproject. In addition to using // these settings, projects that use macroLib should add it as // "compile-internal" and "test-internal" dependency, so that the macroLib jar // does not need to be published. For example: // // lazy val subProject = Project(...) // .dependsOn(..., macroLib % "compile-internal, test-internal") // .settings(commonSettings, usesMacros) // lazy val usesMacros = Seq( // Because the macroLib is an internal dependency to projects that use this // setting, the macroLib is not published. But that means we need to copy the // macro src/bin into projects that use it, essentially inlining macros into // the projects that use them. This is standard practice according to: // // https://www.scala-sbt.org/1.x/docs/Macro-Projects.html#Distribution // // Note that for packageBin, we only copy directories and class files--this // ignores files such a META-INFA/LICENSE and NOTICE that are duplicated and // would otherwise cause a conflict. Compile / packageBin / mappings ++= (macroLib / Compile / packageBin / mappings).value .filter { case (f, _) => f.isDirectory || f.getPath.endsWith(".class") }, Compile / packageSrc / mappings ++= (macroLib / Compile / packageSrc / mappings).value ) lazy val libManagedSettings = Seq( genManaged := { (Compile / managedSources).value (Compile / managedResources).value }, Compile / genProps := { val cp = (propgen / Runtime / dependencyClasspath).value val inSrc = (propgen / Runtime / sources).value val inRSrc = (propgen / Compile / resources).value val stream = (propgen / streams).value val outdir = (Compile / sourceManaged).value val mainClass = "org.apache.daffodil.propGen.PropertyGenerator" val args = Seq(mainClass, outdir.toString) val filesToWatch = (inSrc ++ inRSrc).toSet val cachedFun = FileFunction.cached(stream.cacheDirectory / "propgen") { _ => val forkCaptureLogger = ForkCaptureLogger() val forkOpts = ForkOptions() .withOutputStrategy(Some(LoggedOutput(forkCaptureLogger))) .withBootJars(cp.files.toVector) val ret = Fork.java(forkOpts, args) forkCaptureLogger.stderr.foreach { stream.log.error(_) } if (ret != 0) { sys.error("Failed to generate code") } val files = forkCaptureLogger.stdout.map { f => new File(f) }.toSet stream.log.info(s"generated ${files.size} Scala sources to $outdir") files } cachedFun(filesToWatch).toSeq }, Compile / genSchemas := { val inRSrc = (propgen / Compile / resources).value val stream = (propgen / streams).value val outdir = (Compile / resourceManaged).value val filesToWatch = inRSrc.filter { _.isFile }.toSet val cachedFun = FileFunction.cached(stream.cacheDirectory / "schemasgen") { (schemas: Set[File]) => val files = schemas.map { schema => val out = outdir / "org" / "apache" / "daffodil" / "xsd" / schema.getName IO.copyFile(schema, out) out } stream.log.info(s"generated ${files.size} XML schemas to $outdir") files } cachedFun(filesToWatch).toSeq }, Compile / genVersion := { val resourceDir = (Compile / resourceManaged).value val outFile = resourceDir / "org" / "apache" / "daffodil" / "lib" / "VERSION" if (!outFile.exists || IO.read(outFile) != version.value) { // only write the VERSION file if the version has changed. If we always write, then the // mtime changes and sbt thinks it needs to rebuild everything since a resource changed. IO.write(outFile, version.value) } Seq(outFile) }, Compile / sourceGenerators ++= Seq( (Compile / genProps).taskValue ), Compile / resourceGenerators ++= Seq( (Compile / genSchemas).taskValue, (Compile / genVersion).taskValue ) ) lazy val ratSettings = Seq( ratLicenses := Seq( ("BSD2 ", Rat.BSD2_LICENSE_NAME, Rat.LICENSE_TEXT_PASSERA) ), ratLicenseFamilies := Seq( Rat.BSD2_LICENSE_NAME ), ratExcludes := Rat.excludes, ratFailBinaries := true ) /** * Filter to include only the doc files in our supported API classes * * @param sources - the sequence of files to filter * @return - the filtered sequence of files */ def apiDocSourceFilter(sources: Seq[File]): Seq[File] = sources.filter { source => val str = source.toString val oad = "/org/apache/daffodil" lazy val excludedForJAPI = str.contains(oad + "/japi/") && { // Some things are excluded from the JAPI javadoc because they are internal, non-API // These cannot be excluded from SAPI because symbols-not-found issues with scaladoc. str.contains("$") || str.contains("packageprivate") } lazy val included = { str.contains(oad + "/udf/") || str.contains(oad + "/sapi/") || str.contains(oad + "/japi/") || str.contains(oad + "/runtime1/layers/api/") // // There are files in runtime1/api that are NOT part of the public, supported API. // I tried to include all of runtime1/api, and exclude those files, but could not // get that to work, so now we include individually each file that is part of the // published runtime1 API // // NOTE: Commented out for now. genjavadoc doesn't handle the traits in // these files, so for now these are undocumented. // // FIXME: DAFFODIL-2902 // str.contains(oad + "/runtime1/api/DFDLPrimType") || // str.contains(oad + "/runtime1/api/Infoset") || // str.contains(oad + "/runtime1/api/Metadata") } val res = included && !excludedForJAPI res } lazy val unidocSettings = Seq( ScalaUnidoc / unidoc / unidocProjectFilter := inProjects(sapi, udf, runtime1), ScalaUnidoc / unidoc / scalacOptions := Seq( "-doc-title", "Apache Daffodil " + version.value + " Scala API", "-doc-root-content", (sapi / baseDirectory).value + "/root-doc.txt" ), ScalaUnidoc / unidoc / unidocAllSources := (ScalaUnidoc / unidoc / unidocAllSources).value.map(apiDocSourceFilter), JavaUnidoc / unidoc / unidocProjectFilter := inProjects(japi, udf, runtime1), JavaUnidoc / unidoc / javacOptions := Seq( "-windowtitle", "Apache Daffodil " + version.value + " Java API", "-doctitle", "<h1>Apache Daffodil " + version.value + " Java API</h1>", "-notimestamp", "-quiet" ), JavaUnidoc / unidoc / unidocAllSources := (JavaUnidoc / unidoc / unidocAllSources).value.map(apiDocSourceFilter) ) lazy val genTunablesDocSettings = Seq( Compile / genTunablesDoc := { val stream = (propgen / streams).value val dafExtFile = (propgen / Compile / resources).value.find(_.getName == "dafext.xsd").get val outputDocFile = (Compile / target).value / "tunables.md" // parse xsd file val dafExtXml = scala.xml.XML.loadFile(dafExtFile) // extract tunables information val tunablesElements = (dafExtXml \ "element").filter(_ \@ "name" == "tunables") \\ "all" \ "element" // build documentation val documentationTuple = tunablesElements.map { ele => val subtitle = ele \@ "name" val documentation = (ele \ "annotation" \ "documentation").text.trim.split("\n").map(_.trim).mkString("\n") val default = ele \@ "default" (subtitle, documentation, default) } val (deprecated, nonDeprecated) = documentationTuple.partition { t => t._2.startsWith("Deprecated") } if (nonDeprecated.isEmpty) throw new MessageOnlyException( "tunables document generation failed as non-deprecated elements list is empty" ) val nonDeprecatedDefs = nonDeprecated.map { case (sub, desc, default) => s""" |#### $sub |$desc | |default: $default |""".stripMargin } val deprecatedList = deprecated.map(_._1) val documentationPage = s"""|--- |layout: page |title: Tunables |group: nav-right |--- |<!-- |{% comment %} |Licensed to the Apache Software Foundation (ASF) under one or more |contributor license agreements. See the NOTICE file distributed with |this work for additional information regarding copyright ownership. |The ASF licenses this file to you under the Apache License, Version 2.0 |(the "License"); you may not use this file except in compliance with |the License. You may obtain a copy of the License at | |http://www.apache.org/licenses/LICENSE-2.0 | |Unless required by applicable law or agreed to in writing, software |distributed under the License is distributed on an "AS IS" BASIS, |WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |See the License for the specific language governing permissions and |limitations under the License. |{% endcomment %} |--> |<!-- |{% comment %} |This file is generated using ``sbt genTunablesDoc``. Update that task in Daffodil to update this file. |{% endcomment %} |--> | |Daffodil provides tunables as a way to change its behavior. |Tunables are set by way of the ``tunables`` element in [config files](/configuration) |or from the [cli](/cli) via the ``-T`` option. | |#### Config Example | ``` xml | <daf:dfdlConfig | xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext"> | <daf:tunables> | <daf:suppressSchemaDefinitionWarnings> | encodingErrorPolicyError | </daf:suppressSchemaDefinitionWarnings> | </daf:tunables> |</daf:dfdlConfig> | ``` | | The config file can then be passed into daffodil subcommands via the ``-c|--config`` options. | |#### CLI Example | ``` bash | daffodil parse -s schema.xsd -TsuppressSchemaDefinitionWarnings="encodingErrorPolicyError" data.bin | ``` | | |### Definitions |${nonDeprecatedDefs.mkString("\n")} | |### Deprecated |${deprecatedList.mkString("- ", "\n- ", "")} |""".stripMargin IO.write(outputDocFile, s"$documentationPage") stream.log.info(s"generated tunables documentation at: $outputDocFile") Seq(outputDocFile) } ) lazy val genCExamplesSettings = Seq( Compile / genCExamples := { val cp = (codeGenC / Test / dependencyClasspath).value val inSrc = (codeGenC / Compile / sources).value val inRSrc = (codeGenC / Test / resources).value val stream = (codeGenC / streams).value val filesToWatch = (inSrc ++ inRSrc).toSet val cachedFun = FileFunction.cached(stream.cacheDirectory / "genCExamples") { _ => val forkCaptureLogger = ForkCaptureLogger() val forkOpts = ForkOptions() .withOutputStrategy(Some(LoggedOutput(forkCaptureLogger))) .withBootJars(cp.files.toVector) val mainClass = "org.apache.daffodil.codegen.c.DaffodilCExamplesGenerator" val outdir = (codeGenC / Test / sourceDirectory).value / "examples" val args = Seq(mainClass, outdir.toString) val ret = Fork.java(forkOpts, args) forkCaptureLogger.stderr.foreach { stream.log.error(_) } if (ret != 0) { sys.error("failed to generate C example files") } val files = forkCaptureLogger.stdout .filterNot(_.startsWith("WARNING")) .map { f => new File(f) } .toSet stream.log.info(s"generated ${files.size} C examples to $outdir") files } cachedFun(filesToWatch).toSeq }, Compile / compile := { val res = (Compile / compile).value (Compile / genCExamples).value res } )