#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This is the background worker, that scans for abusive behavior and logs it as new bans, according to the rule-sets. """ import re import netaddr import json import time import datetime import plugins.database import plugins.openapi import os import hashlib import socket DEBUG = False # If True, no DB writes/deletes will be done def make_sha1(t, encoding = 'utf-8'): return hashlib.sha1(t.encode(encoding)).hexdigest() def to_block(ipaddress): """ Converts an IP address or CIDR block to an IPNetwork object """ block = None try: if '/' in ipaddress: block = netaddr.IPNetwork(ipaddress) else: if ':' in ipaddress: # IPv6? block = netaddr.IPNetwork("%s/128" % ipaddress) else: # IPv4? block = netaddr.IPNetwork("%s/32" % ipaddress) except netaddr.AddrFormatError: raise plugins.openapi.BlockyHTTPError(400, "Invalid IP address or network!") return block def addnote(DB, ntype = 'automatic', what = "No message supplied"): doc = { 'epoch': int(time.time()), 'ntype': ntype, 'text': what, } DB.ES.index(index=DB.dbname, doc_type = 'note', body = doc) def get_whitelist(DB): """ Get the entire whitelist """ whitelist = [] res = DB.ES.search( index=DB.dbname, doc_type="whitelist", size = 5000, body = { 'query': { 'match_all': {} } } ) for hit in res['hits']['hits']: doc = hit['_source'] ipaddress = doc.get('ip') if ipaddress: ipaddress = ipaddress.strip() # blocky/1 bug timeout = doc.get('timeout') # If there is a timeout in a whitelist object, check if we need to remove it if timeout and timeout > 0: if timeout < time.time(): addnote(DB, 'cleanup', "Removing whitelist rule %s for %s (timed out)" % (hit['_id'], ipaddress)) DB.ES.delete(index=DB.dbname, doc_type = 'whitelist', id = hit['_id']) # convert to IPNetwork object block = to_block(ipaddress) if block: whitelist.append(block) return whitelist def get_banlist(DB): """ Get the entire banlist """ banlist = [] res = DB.ES.search( index=DB.dbname, doc_type="ban", size = 10000, body = { 'query': { 'match_all': {} } } ) for hit in res['hits']['hits']: doc = hit['_source'] ipaddress = doc.get('ip') if not ipaddress: ipaddress = hit['_id'].replace('_', '/') # Blocky/1 syntax, bah if ipaddress: ipaddress = ipaddress.strip() # blocky/1 bug block = to_block(ipaddress) if block: banlist.append(block) return banlist def get_iptables(DB): """ Get the entire list of host iptable sets """ rulelist = [] res = DB.ES.search( index=DB.dbname, doc_type="iptables", size = 5000, body = { 'query': { 'match_all': {} } } ) for hit in res['hits']['hits']: doc = hit['_source'] hostname = doc.get('hostname', 'unknown') rules = doc.get('iptables', []) for rule in rules: rule['ip'] = to_block(rule['source']) entry = { 'hostname': hostname, 'rules': rules, } rulelist.append(entry) return rulelist def construct_query(doctype, query, initial_terms = []): """ Construct an ES query based on doctype and ruleset """ terms = initial_terms nterms = [] for term in query: numm = re.match(r"(\S+)=([0-9.]+)", term) strm = re.match(r"(\S+)=\"(.+)\"", term) nstrm = re.match(r"(\S+)\!=\"(.+)\"", term) if numm: terms.append({"term": {numm.group(1): int(numm.group(2))}}) elif nstrm: if '*' in nstrm.group(2): nterms.append({"match": {nstrm.group(1): nstrm.group(2)}}) else: nterms.append({"term": {nstrm.group(1): nstrm.group(2)}}) elif strm: if '*' in strm.group(2): terms.append({"match": {strm.group(1): strm.group(2)}}) else: terms.append({"term": {strm.group(1): strm.group(2)}}) # Now construct the final query q = None if doctype == 'httpd_visits': q = { "aggregations": { "byip": { "filter": { "bool": { "must": terms, "must_not": nterms } }, "aggs": { "clients": { "terms": { "field": "clientip.keyword", "size": 50, "order": { "_count": "desc" } } } } } }, "size": 0 } elif doctype == 'httpd_traffic': q = { "aggregations": { "byip": { "filter": { "bool": { "must": terms, "must_not": nterms, } }, "aggs": { "clients": { "terms": { "field": "clientip.keyword", "size": 50, "order": { "traffic": "desc" } }, "aggs": { "traffic": { "sum": { "field": "bytes" } } } } } } }, "size": 0 } return q def start(DB, config, pidfile): addnote(DB, 'system', "(re)starting Blocky/2...") time.sleep(15) # First, check that indices are present # We're assuming ES6 here... if type(DB.ES) is plugins.database.BlockyESWrapper: for el in ['whitelist', 'banlist', 'rules', 'notes']: if not DB.ES.exists_doctype(index=DB.dbname, doc_type=el): print("DB index %s does not exist, creating.." % el) DB.ES.create(index=DB.dbname, doc_type = el) # Okay, now run the rule-set every once in a while while os.path.exists(pidfile): print("Running rulesets against database...") now = time.time() # First, fetch all rules rules = [] res = DB.ES.search( index=DB.dbname, doc_type="rule", size = 5000, body = { 'query': { 'match_all': {} } } ) for hit in res['hits']['hits']: doc = hit['_source'] doc['rid'] = hit['_id'] rules.append(doc) # Prep list of bad IPs to block bad_ips = [] # Prep list of indices to check against, for performance reasons d = datetime.datetime.utcnow() t = [] for i in range(0,3): t.append(d.strftime("loggy-%Y.%m.%d")) d -= datetime.timedelta(days = 1) threes = ",".join(t) # Past three days # Now, run each rule for rule in rules: limit = rule.get('limit') span = rule.get('span') doctype = rule.get('type') name = rule.get('name', 'Generic rule') query = rule.get('query') rid = rule.get('rid', 'null') if limit and span and query: print("Running rule '%s'..." % name) # Start with timestamp terms terms = [] terms.append({ "range": { "@timestamp": { "gt": ("now-%uh" % span) } } }) # For each term in our blocky query, convert to ES terms q = construct_query(doctype, query, terms) # If valid query, run it and find bad IPs if q: res = DB.ES.search(index=threes, request_timeout=90, body=q) if res and 'aggregations' in res: if doctype == 'httpd_visits': for suspect in res['aggregations']['byip']['clients']['buckets']: c = suspect['doc_count'] i = suspect['key'] if c > limit: r = "%s (%u >= limit of %u)" % (name, c, limit) bad_ips.append({'ip': i, 'reason': r, 'target': '*', 'rid': rid, 'epoch': int(time.time())}) print("Found offender: %s; %s" % (i, r)) elif doctype == 'httpd_traffic': for suspect in res['aggregations']['byip']['clients']['buckets']: c = suspect['traffic']['value'] i = suspect['key'] if c > limit: r = "%s (%u >= limit of %u)" % (name, c, limit) bad_ips.append({'ip': i, 'reason': r, 'target': '*', 'rid': rid, 'epoch': int(time.time())}) print("Found offender: %s; %s" % (i, r)) print("Done with rules after %u seconds, found %u offenders" % (time.time() - now, len(bad_ips))) # Now we have a list of bad IPs. # Compare against whitelist, filter out any that are there # Block the rest # Fetch whitelist whitelist = get_whitelist(DB) # For each baddie, compare against whitelist to_ban = [] for bad_ip in bad_ips: ipaddress = netaddr.IPAddress(bad_ip['ip']) whitelisted = False for whiteblock in whitelist: if ipaddress in whiteblock: print("%s is whitelisted as %s, ignoring offenses..." % (ipaddress, whiteblock)) whitelisted = True break if not whitelisted: to_ban.append(bad_ip) # For each IP we should be banning, ban if not already banned for bad_ip in to_ban: banid = make_sha1(bad_ip['ip']) if not DB.ES.exists(index=DB.dbname, doc_type = 'ban', id = banid): rdns = bad_ip['ip'] try: rdns = socket.gethostbyaddr(bad_ip['ip'])[0] except: pass # don't care, at all bad_ip['dns'] = rdns if rdns else bad_ip['ip'] print("Banning %s as %s" % (bad_ip['ip'], banid)) if not DEBUG: addnote(DB, 'autoban', "Banning %s on %s as %s (%s)" % (bad_ip['ip'], bad_ip['target'], banid, bad_ip['reason'])) DB.ES.index(index=DB.dbname, doc_type = 'ban', id = banid, body = bad_ip) # Now sleep for a minute time.sleep(120)