glue/greentaxi_etl.py [31:54]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        ("passenger_count", "long", "passenger_count", "long"),
        ("trip_distance", "double", "trip_distance", "double"),
        ("ratecodeid", "long", "ratecodeid", "long"),
        ("store_and_fwd_flag", "string", "store_and_fwd_flag", "string"),
        ("pulocationid", "long", "pulocationid", "long"),
        ("dolocationid", "long", "dolocationid", "long"),
        ("payment_type", "long", "payment_type", "long"),
        ("fare_amount", "double", "fare_amount", "double"),
        ("extra", "double", "extra", "double"),
        ("mta_tax", "double", "mta_tax", "double"),
        ("tip_amount", "double", "tip_amount", "double"),
        ("tolls_amount", "double", "tolls_amount", "double"),
        ("improvement_surcharge", "double", "improvement_surcharge", "double"),
        ("total_amount", "double", "total_amount", "double"),
        ("congestion_surcharge", "double", "congestion_surcharge", "double"),
        ("partition_0", "string", "partition_0", "string"),
    ],
    transformation_ctx="ApplyMapping_node2",
)

# Script to add column
##Custom Transformation
#convert to a Spark DataFrame...
customDF = ApplyMapping_node2.toDF()
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



glue/yellowtaxi_etl.py [31:54]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        ("passenger_count", "long", "passenger_count", "long"),
        ("trip_distance", "double", "trip_distance", "double"),
        ("ratecodeid", "long", "ratecodeid", "long"),
        ("store_and_fwd_flag", "string", "store_and_fwd_flag", "string"),
        ("pulocationid", "long", "pulocationid", "long"),
        ("dolocationid", "long", "dolocationid", "long"),
        ("payment_type", "long", "payment_type", "long"),
        ("fare_amount", "double", "fare_amount", "double"),
        ("extra", "double", "extra", "double"),
        ("mta_tax", "double", "mta_tax", "double"),
        ("tip_amount", "double", "tip_amount", "double"),
        ("tolls_amount", "double", "tolls_amount", "double"),
        ("improvement_surcharge", "double", "improvement_surcharge", "double"),
        ("total_amount", "double", "total_amount", "double"),
        ("congestion_surcharge", "double", "congestion_surcharge", "double"),
        ("partition_0", "string", "partition_0", "string"),
    ],
    transformation_ctx="ApplyMapping_node2",
)

# Script to add column
##Custom Transformation
#convert to a Spark DataFrame...
customDF = ApplyMapping_node2.toDF()
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -