lambda-functions/cloudfront-logs-processor-function/prep-data.py [56:143]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                    json_data = json.dumps(json_data)

# cloudfront columns
#           - Name: logdate - 0
#             Type: date
#           - Name: logtime - 1
#             Type: string
#           - Name: location - 2
#             Type: string
#           - Name: bytes - 3
#             Type: bigint
#           - Name: requestip - 4
#             Type: string
#           - Name: method - 5
#             Type: string
#           - Name: host - 6
#             Type: string
#           - Name: uri - 7
#             Type: string
#           - Name: status - 8
#             Type: bigint
#           - Name: referrer - 9
#             Type: string
#           - Name: useragent - 10
#             Type: string
#           - Name: uriquery - 11
#             Type: string
#           - Name: cookie - 12
#             Type: string
#           - Name: resulttype - 13
#             Type: string
#           - Name: requestid - 14
#             Type: string
#           - Name: header - 15
#             Type: string
#           - Name: csprotocol - 16
#             Type: string
#           - Name: csbytes - 17
#             Type: string
#           - Name: timetaken - 18
#             Type: bigint
#           - Name: forwardedfor - 19
#             Type: string
#           - Name: sslprotocol - 20
#             Type: string
#           - Name: sslcipher - 21
#             Type: string
#           - Name: responseresulttype - 22
#             Type: string
#           - Name: protocolversion - 23
#             Type: string
#           - Name: fle-status - 24
#             Type: string
#           - Name: fle-encrypted-fields - 25
#             Type: string
#           - Name: browserfamily - 26
#             Type: string
#           - Name: osfamily - 27
#             Type: string
#           - Name: isbot - 28
#             Type: string
#           - Name: filename - 29
#             Type: string
#           - Name: distribution - 30
#             Type: string
#           - Name: cdn_source - 31
#             Type: string

                    # line = line + '\t' + str(ua.browser.family) + '\t' + str(ua.os.family) + '\t' + str(ua.is_bot) + '\t' + key + '\t' + dist_name + '\t' + cdn_source + '\n'
                    processed = True

                    print("pushing item to firehose stream -> ")
                    print(json_data)

                    firehose.put_record(
                        DeliveryStreamName=delivery_stream,
                            Record={
                                "Data":json_data+"\n"
                            }
                    )
                except Exception as e:
                    print(e)
                    print("Exception during utf8 conversion")

    except Exception as e:
        print(e)
        print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
        raise e
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



lambda-functions/fastly-logs-processor-function/prep-data.py [60:84]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                    json_data = json.dumps(json_data)

# fastly columns
#<134>2019-01-06T04:17:46Z cache-bom18223 S3LogBucket[22354]: 49.36.1.187 "-" "-" [06/Jan/2019:04:17:45 +0000] "GET /sample-videos/hls/TearsOfSteel.m3u8 HTTP/1.1" 403 243 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" 0 "MISS" "33c478dd-0491-4bfb-b303-64f546b819e7"

                    # line = line + '\t' + str(ua.browser.family) + '\t' + str(ua.os.family) + '\t' + str(ua.is_bot) + '\t' + key + '\t' + dist_name + '\t' + cdn_source + '\n'
                    processed = True

                    print("pushing item to firehose stream -> ")
                    print(json_data)

                    firehose.put_record(
                        DeliveryStreamName=delivery_stream,
                            Record={
                                "Data":json_data+"\n"
                            }
                    )
                except Exception as e:
                    print(e)
                    print("Exception during utf8 conversion")

    except Exception as e:
        print(e)
        print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
        raise e
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -