troubleshooting/tools/color-logger/large.py

import datetime
from collections import OrderedDict
import time
import os
import json 
import signal
import random
from fluent import sender

# for local fluent
logger = sender.FluentSender('large-logs-tester')

name = os.environ.get("NAME", "logger")

def print_log(obj):
    if os.environ.get("FLUENT_LOGGER", False):
        logger.emit(name, obj)
        if not logger.emit('large-logs-tester', obj):
            print(logger.last_error)
            logger.clear_last_error() # clear stored error after handled errors
    else:
        print(json.dumps(obj))


PAYLOAD_SIZE = int(os.environ.get("PAYLOAD_SIZE", 32))
PAYLOAD_COUNT = int(os.environ.get("PAYLOAD_COUNT", 100))
MESSAGE_INTERVAL = float(os.environ.get("MESSAGE_INTERVAL", 10))
SIGNAL_PAYLOAD_SIZE = int(os.environ.get("SIGNAL_PAYLOAD_SIZE", 128000))
SIGNAL_PAYLOAD_COUNT = int(os.environ.get("SIGNAL_PAYLOAD_COUNT", 100))

id = 0

def sig_handler(i):
    global id
    for _ in range(10):
        if id < SIGNAL_PAYLOAD_COUNT:
            event_id = int(random.random()*1000)
            print_log(OrderedDict({
                "payload": "x" * SIGNAL_PAYLOAD_SIZE,
                "event_id": event_id,
                "counter": id,
                "global_counter": i, 
                "time": '%s' % datetime.datetime.now(),
            }))
        id += 1
    
def main():
    for i in range(PAYLOAD_COUNT):
        sig_handler(i)
        print("Logged 10 events")
        time.sleep(MESSAGE_INTERVAL)

if __name__ == '__main__': main()