in src/main/scala/deequ/deequ-analysis-verification-runner.scala [244:272]
def createCheckClass(checksDF: DataFrame, dataDF: DataFrame): Seq[DataFrame] = {
checksDF.collect.map { row =>
var checkValidationCode =
"_root_.com.amazon.deequ.checks.Check(_root_.com.amazon.deequ.checks.CheckLevel.Error, \"Review Check\")" + row
.mkString("@")
.split("@")(0)
checkValidationCode = checkValidationCode.replace(
"ConstrainableDataTypes",
"_root_.com.amazon.deequ.constraints.ConstrainableDataTypes"
)
val _checks =
toolbox
.eval(toolbox.parse(checkValidationCode))
.asInstanceOf[Check]
val verificationResult: VerificationResult = {
VerificationSuite()
.onData(dataDF)
.addCheck(_checks)
.run()
}
checkResultsAsDataFrame(spark, verificationResult)
}
}