def extractSuggestionsFromDynamo()

in src/main/scala/deequ/deequ-analysis-verification-runner.scala [163:184]

• 17 lines of code
• 2 McCabe index (conditional complexity)

  def extractSuggestionsFromDynamo(dynoTable: String, dbName: String, table: String): DataFrame = {

    val client = AmazonDynamoDBClientBuilder.standard().build()
    val dynamoDB: DynamoDB = new DynamoDB(client)
    val tableSuggestions: Table = dynamoDB.getTable(dynoTable)
    val index: Index = tableSuggestions.getIndex("table-index")

    val querySpec: QuerySpec = new QuerySpec()
    querySpec.withKeyConditionExpression("tableHashKey = :v_table").withValueMap(new ValueMap().withString(":v_table", dbName + "-" + table))

    val items = index.query(querySpec)
    val iterator = items.iterator()

    var listSuggestions = List.empty[String]
    while (iterator.hasNext()) {
        listSuggestions = listSuggestions :+ iterator.next().toJSON()
    }

    val rddDynamoSuggestions = spark.sparkContext.parallelize(listSuggestions)
    val dfDynamoSuggestions = spark.read.json(rddDynamoSuggestions)
    return dfDynamoSuggestions
  }