in loggy.py [0:0]
def run(self):
iname = time.strftime(INDEX_PATTERN)
if iname not in self.parent.indices:
self.parent.indices.append(iname)
if not self.parent.elastic.indices.exists(index=iname):
mappings = {}
for name, entry in self.parent.config.get("rawfields").items():
map_js: typing.Dict[str, typing.Dict[typing.Union[str, bool]]] = {
"_all": {"enabled": True},
"properties": {
"@timestamp": {"store": True, "type": "date", "format": "yyyy/MM/dd HH:mm:ss"},
"@node": {"store": True, "type": "keyword"},
"status": {"store": True, "type": "long"},
"date": {"store": True, "type": "keyword"},
"geo_location": {"type": "geo_point", "geohash": True},
},
}
for field in entry.split(","):
x = field.strip()
map_js["properties"][x] = {"store": True, "type": "keyword"}
mappings[entry] = map_js
if not DEBUG:
self.parent.elastic.indices.create(
index=iname,
ignore=400,
body={
"settings": {
"index.mapping.ignore_malformed": True,
"number_of_shards": 2,
"number_of_replicas": 0,
},
"mappings": mappings,
},
)
else:
print(mappings)
js_arr = []
for js in self.json:
# GeoHash conversion
if "geo_lat" in js and "geo_long" in js:
try:
js["geo_location"] = {"lat": float(js["geo_lat"]), "lon": float(js["geo_long"])}
except ValueError:
pass
js["@version"] = 3
js["@timestamp"] = time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime())
js["host"] = self.parent.logger.nodename
js["@node"] = self.parent.logger.nodename
if FINGERPRINT:
js["@fingerprint"] = FINGERPRINT
js["@fingerprint_sha"] = FINGERPRINT_SHA
# Rogue string sometimes, we don't want that!
if "bytes" in js:
try:
js["bytes"] = int(js["bytes"])
except ValueError:
js["bytes"] = 0
if "request" in js and "url" not in js:
match = re.match(r"(GET|POST)\s+(.+)\s+HTTP/.+", js["request"])
if match:
js["url"] = match.group(2)
if "bytes" in js and isinstance(js["bytes"], str) and js["bytes"].isdigit():
js["bytes_int"] = int(js["bytes"])
js_arr.append({"_op_type": "index", "_index": iname, "doc": js, "_source": js})
if len(js_arr) > 0:
if DEBUG:
print(js_arr)
else:
try:
elasticsearch.helpers.bulk(self.parent.elastic, js_arr)
except elasticsearch.helpers.BulkIndexError as e:
print(e)