def save_tokenizer()

in train/train.py [0:0]

• 8 lines of code
• 1 McCabe index (conditional complexity)

def save_tokenizer(tokenizer):
    s3_client = boto3.client('s3')
    s3_sourcedir = os.environ['SM_MODULE_DIR']
    s3_bucket = s3_sourcedir.split("/")[2]
    array_data = io.BytesIO()
    pickle.dump(tokenizer,array_data)
    array_data.seek(0)
    s3_client.upload_fileobj(array_data,s3_bucket,"toxic_comments/tokenizer.pkl")