in tools/archiver.py [0:0]
def archive_message(self, args, mlist, msg, raw_message):
"""Send the message to the archiver.
:param args: Command line args (dry, dump)
:param mlist: The IMailingList object.
:param msg: The message object.
:param raw_message: Raw message bytes
:return (lid, mid)
"""
lid = normalize_lid(mlist.list_id)
private = False
if hasattr(mlist, 'archive_public') and mlist.archive_public == True:
private = False
elif hasattr(mlist, 'archive_public') and mlist.archive_public == False:
private = True
elif hasattr(mlist, 'archive_policy') and mlist.archive_policy is not ArchivePolicy.public:
private = True
ojson, contents, msg_metadata, irt = self.compute_updates(lid, private, msg)
if not ojson:
_id = msg.get('message-id') or msg.get('Subject') or msg.get("Date")
raise Exception("Could not parse message %s for %s" % (_id,lid))
if args.dry:
print("**** Dry run, not saving message to database *****")
return lid, ojson['mid']
try:
if contents:
for key in contents:
self.index(
index=self.dbname,
doc_type="attachment",
id=key,
body = {
'source': contents[key]
}
)
self.index(
index=self.dbname,
doc_type="mbox",
id=ojson['mid'],
consistency = self.consistency,
body = ojson
)
self.index(
index=self.dbname,
doc_type="mbox_source",
id=ojson['mid'],
consistency = self.consistency,
body = {
"message-id": msg_metadata['message-id'],
"source": mbox_source(raw_message)
}
)
# If we have a dump dir and ES failed, push to dump dir instead as a JSON object
# We'll leave it to another process to pick up the slack.
except Exception as err:
if args.dump:
print("Pushing to ES failed, but dumponfail specified, dumping JSON docs")
uid = uuid.uuid4()
mboxPath = os.path.join(args.dump, "%s.json" % uid)
with open(mboxPath, "w") as f:
json.dump({
'id': ojson['mid'],
'mbox': ojson,
'mbox_source': {
"message-id": msg_metadata['message-id'],
"source": mbox_source(raw_message)
},
'attachments': contents
},f , indent = 2)
f.close()
sys.exit(0) # We're exiting here, the rest can't be done without ES
# otherwise fail as before
raise err
# If MailMan and list info is present, save/update it in ES:
if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description and mlist.list_name:
self.index(
index=self.dbname,
doc_type="mailinglists",
id=lid,
consistency = self.consistency,
body = {
'list': lid,
'name': mlist.list_name,
'description': mlist.description,
'private': private
}
)
if logger:
logger.info("Pony Mail archived message %s successfully", ojson['mid'])
oldrefs = []
# Is this a direct reply to a pony mail email?
if irt != "":
dm = re.search(r"pony-([a-f0-9]+)-([a-f0-9]+)@", irt)
if dm:
cid = dm.group(1)
mid = dm.group(2)
if self.es.exists(index=self.dbname, doc_type='account', id=cid):
doc = self.es.get(index=self.dbname, doc_type='account', id=cid)
if doc:
oldrefs.append(cid)
# N.B. no index is supplied, so ES will generate one
self.index(
index=self.dbname,
doc_type="notifications",
consistency = self.consistency,
body = {
'type': 'direct',
'recipient': cid,
'list': lid,
'private': private,
'date': ojson['date'],
'from': msg_metadata['from'],
'to': msg_metadata['to'],
'subject': msg_metadata['subject'],
'message-id': msg_metadata['message-id'],
'in-reply-to': irt,
'epoch': ojson['epoch'],
'mid': mid,
'seen': 0
}
)
if logger:
logger.info("Notification sent to %s for %s", cid, mid)
# Are there indirect replies to pony emails?
if msg_metadata.get('references'):
for im in re.finditer(r"pony-([a-f0-9]+)-([a-f0-9]+)@", msg_metadata.get('references')):
cid = im.group(1)
mid = im.group(2)
if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
# does the user want to be notified of indirect replies?
if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications') == 'indirect' and not cid in oldrefs:
oldrefs.append(cid)
# N.B. no index is supplied, so ES will generate one
self.index(
index=self.dbname,
consistency = self.consistency,
doc_type="notifications",
body = {
'type': 'indirect',
'recipient': cid,
'list': lid,
'private': private,
'date': ojson['date'],
'from': msg_metadata['from'],
'to': msg_metadata['to'],
'subject': msg_metadata['subject'],
'message-id': msg_metadata['message-id'],
'in-reply-to': irt,
'epoch': ojson['epoch'],
'mid': mid,
'seen': 0
}
)
if logger:
logger.info("Notification sent to %s for %s", cid, mid)
return lid, ojson['mid']