def infer()

in gluejobs/n1_c360_dispositions.py [0:0]

• 26 lines of code
• 3 McCabe index (conditional complexity)

def infer(location,tablename,databasename):
    # spark = SparkSession.builder.appName("spark").config("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version", "2").config("spark.speculation", "false").config("hive.exec.dynamic.partition", "true").config("hive.exec.dynamic.partition.mode","nonstrict").enableHiveSupport().getOrCreate()
    df = spark.read.format("parquet").parquet(location)

    cols = df.dtypes
    buf = []
    buf.append('CREATE EXTERNAL TABLE '+databasename+'.'+tablename+' ( \n')
    keyanddatatypes =  df.dtypes
    sizeof = len(df.dtypes)
    print ('size----------',sizeof)
    count=1;
    for eachvalue in keyanddatatypes:
        # print (count,sizeof,eachvalue)
        if count == sizeof:
            total = '`'+str(eachvalue[0])+str('`   ')+str(eachvalue[1])
        else:
            total =  '`'+str(eachvalue[0]) + str('`  ') + str(eachvalue[1]) + str(',')
        buf.append(total+'\n')
        count = count + 1

    buf.append(' )')
    buf.append(' STORED as parquet ')
    buf.append(" LOCATION ")
    buf.append("'")
    buf.append(location)
    buf.append("'")
    ##partition by pt
    tabledef = ''.join(buf)

    print ("---------print definition ---------")
    print (tabledef)

    ddl(tabledef)