def lambda_handler()

in ingest_suricata_rules/rules_processor.py [0:0]

• 64 lines of code
• 18 McCabe index (conditional complexity)

def lambda_handler(event, context):
    bucket = event['Records'][0]['s3']['bucket']['name']
    key = event['Records'][0]['s3']['object']['key']


    # bucket = os.environ["SuricataRulesBucket"]
    # key = "rules/emerging-imap.rules"  
    
    suricata_rules_filename = key.split('/')[-1].split('.')[0].replace('_','-')
    s3.download_file(bucket, key, '/tmp/'+suricata_rules_filename)
    
    content=parse_file('/tmp/'+suricata_rules_filename) 
    print("Processing Suricata Rules file : s3://"+bucket+"/"+key)
    print("Initial Suricata Rules count including commented rules (excluding commented headers): ", len(content))
    # Drop commented / disabled rules
    initial_ruleset = [ rule for rule in content if rule.enabled == True ]
    print("Initial Suricata Rules count (uncommented rule lines): ", len(initial_ruleset))

    # When testing with 3000+ rules faced lamdba timeout issues
    # If file contains more than 1000 active rules split into multiple files to avoid lamdba timeout issues
    
    if len(initial_ruleset) > 1000:
        print("Initial Rule Count > 1000. Spliting files by 1000 rules to process them individually")
        split_files(initial_ruleset,1000,suricata_rules_filename,bucket)
        exit()

    valid_ruleset, dropped_ruleset = process_ruleset(initial_ruleset)
    valid_ruleset, dropped_ruleset = replace_rule_vars_with_values(valid_ruleset,dropped_ruleset)
    # Drop Rules with undefined RuleVariables
    final_ruleset = [ rule for rule in valid_ruleset if rule.enabled == True ]
    final_rule_count = len(final_ruleset)
    print("Valid Rule count that can be applied to ANF: ", final_rule_count)
    print("Dropped Rule count that cannot be applied to ANF: ", len(dropped_ruleset))
    
    save_rulesets(final_ruleset,bucket,suricata_rules_filename,'FINALBEFOREAPPLY')
    save_rulesets(dropped_ruleset,bucket,suricata_rules_filename,'DROPPED')
    
    # Calculate required rule group capacity
    if final_rule_count == 0 :
        print("SKIPPING RULE FILE: No Valid Rules found that can be applied to ANF for file :", suricata_rules_filename)
        save_rulesets(final_ruleset,bucket,suricata_rules_filename,'EMPTY')
        exit()
    
    # if 1 <= final_rule_count <= 80:
    #     RuleGroupCapacity = 100
    # if 81 <= final_rule_count <= 400:
    #     RuleGroupCapacity = 500
    # if final_rule_count > 401:
    #     RuleGroupCapacity = 1000
    
    # Roundup to nearest hundred with 30% buffer
    RuleGroupCapacity = int(math.ceil((final_rule_count * 1.3) / 100.0)) * 100
    
    # Create/Update RuleGroup per each ruleset chunk
    RuleGroupName="suricata-"+suricata_rules_filename
    RulesString = "\n".join([ str(rule) for rule in final_ruleset ])

    # To convert any IDS Rulegroup to IPS Rulegroup
    # Lambda checks ssm param : ConvertRuleGroupIDStoIPS for list of suricata rulegroups to create/update respective ips rulegroups
    ids_ips_convert_list = [x.strip() for x in get_rulegroups_to_convert(ConvertRuleGroupIDStoIPSSSMParam).split(",")] 

    if check_rule_group_exists(RuleGroupName):
        RuleGroup=get_rule_group(RuleGroupName)
        UpdateToken=RuleGroup["UpdateToken"]
        print("Attempting Update RuleGroup: "+RuleGroupName+ " with " + str(len(final_ruleset)) + " rules" )
        
        try:
            response = anf.update_rule_group(
                            RuleGroupName=RuleGroupName,
                            Type=RuleGroupType,
                            Rules=RulesString,
                            UpdateToken=RuleGroup["UpdateToken"]
                        )
            print(response)
            save_rulesets(final_ruleset,bucket,RuleGroupName,'APPLIED')
            if RuleGroupName in ids_ips_convert_list:
                trigger_ids_to_ips(final_ruleset,bucket,RuleGroupName)

        except Exception as err:
            save_rulesets(final_ruleset,bucket,RuleGroupName,'ERROR')
            raise err
    else:
        print("Attempting Create RuleGroup: "+RuleGroupName+ " with " + str(len(final_ruleset)) + " rules" )
        try:
            response = anf.create_rule_group(
                            RuleGroupName=RuleGroupName,
                            Type=RuleGroupType,
                            Rules=RulesString,
                            Capacity=RuleGroupCapacity
                        )
            print(response)
            save_rulesets(final_ruleset,bucket,RuleGroupName,'APPLIED')
            if RuleGroupName in ids_ips_convert_list:
                trigger_ids_to_ips(final_ruleset,bucket,RuleGroupName)
        except Exception as err:
            save_rulesets(final_ruleset,bucket,RuleGroupName,'ERROR')
            raise err