#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import importlib.util import logging import os import os.path import shutil import sys if sys.version_info <= (3, 3): print("This script requires Python 3.4 or higher") sys.exit(-1) # Check for all required python packages wanted_pkgs = [ 'elasticsearch',# used by setup.py, archiver.py and elastic.py 'formatflowed', # used by archiver.py 'netaddr', # used by archiver.py 'certifi', # used by archiver.py and elastic.py ] missing_pkgs = list(wanted_pkgs) # copy to avoid corruption for pkg in wanted_pkgs: if importlib.util.find_spec(pkg): missing_pkgs.remove(pkg) if missing_pkgs: print("It looks like you need to install some python modules first") print("The following packages are required: ") for pkg in missing_pkgs: print(" - %s" % pkg) print("You may use your package manager, or run the following command:") print("pip3 install %s" % " ".join(missing_pkgs)) sys.exit(-1) # at this point we can assume elasticsearch is present from elasticsearch import Elasticsearch from elasticsearch import ElasticsearchException from elasticsearch import ConnectionError as ES_ConnectionError from elasticsearch import VERSION as ES_VERSION ES_MAJOR = ES_VERSION[0] # CLI arg parsing parser = argparse.ArgumentParser(description='Command line options.') parser.add_argument('--defaults', dest='defaults', action='store_true', help='Use default settings') parser.add_argument('--dbprefix', dest='dbprefix') parser.add_argument('--clobber', dest='clobber', action='store_true', help='Allow overwrite of ponymail.cfg & ../site/api/lib/config.lua (default: create *.tmp if either exists)') parser.add_argument('--dbhost', dest='dbhost', type=str, help='ES backend hostname') parser.add_argument('--dbport', dest='dbport', type=str, help='DB port') parser.add_argument('--dbname', dest='dbname', type=str, help='ES DB name') parser.add_argument('--dbshards', dest='dbshards', type=int, help='DB Shard Count') parser.add_argument('--dbreplicas', dest='dbreplicas', type=int, help='DB Replica Count') parser.add_argument('--mailserver', dest='mailserver', type=str, help='Host name of outgoing mail server') parser.add_argument('--mldom', dest='mldom', type=str, help='Domains to accept mail for via UI') parser.add_argument('--wordcloud', dest='wc', action='store_true', help='Enable word cloud') parser.add_argument('--skiponexist', dest='soe', action='store_true', help='Skip setup if ES index exists') parser.add_argument('--noindex', dest='noi', action='store_true', help="Don't make an ES index, assume it exists") parser.add_argument('--nocloud', dest='nwc', action='store_true', help='Do not enable word cloud') parser.add_argument('--generator', dest='generator', type=str, help='Document ID Generator to use (legacy, medium, cluster, full)') args = parser.parse_args() print("Welcome to the Pony Mail setup script!") print("Let's start by determining some settings...") print("") hostname = "" port = 0 dbname = "" mlserver = "" mldom = "" wc = "" genname = "" wce = False shards = 0 replicas = -1 urlPrefix = None # If called with --defaults (like from Docker), use default values if args.defaults: hostname = "localhost" port = 9200 dbname = "ponymail" mlserver = "localhost" mldom = "example.org" wc = "Y" wce = True shards = 1 replicas = 0 genname = "cluster" urlPrefix = '' # Accept CLI args, copy them if args.dbprefix: urlPrefix = args.dbprefix if args.dbhost: hostname = args.dbhost if args.dbport: port = int(args.dbport) if args.dbname: dbname = args.dbname if args.mailserver: mlserver = args.mailserver if args.mldom: mldom = args.mldom if args.wc: wc = args.wc if args.nwc: wc = False if args.dbshards: shards = args.dbshards if args.dbreplicas: replicas = args.dbreplicas if args.generator: genname = args.generator while hostname == "": hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ") while urlPrefix == None: urlPrefix = input("Database URL prefix if any (hit enter if none): ") while port < 1: try: port = int(input("What port is ElasticSearch listening on? (normally 9200): ")) except ValueError: pass while dbname == "": dbname = input("What would you like to call the mail index (e.g. ponymail): ") while mlserver == "": mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ") while mldom == "": mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ") while wc == "": wc = input("Would you like to enable the word cloud feature? (Y/N): ") if wc.lower() == "y": wce = True while genname == "": gens = ['legacy', 'medium', 'cluster', 'full'] print ("Please select a document ID generator:") print("1 LEGACY: The original document generator for v/0.1-0.8 (no longer recommended)") print("2 MEDIUM: The medium comprehensive generator for v/0.9 (no longer recommended)") print("3 REDUNDANT: Near-full message digest, discard MTA trail (recommended for clustered setups)") print("4 FULL: Full message digest with MTA trail (recommended for single-node setups).") try: gno = int(input("Please select a generator [1-4]: ")) if gno <= len(gens) and gens[gno-1]: genname = gens[gno-1] except ValueError: pass while shards < 1: try: shards = int(input("How many shards for the ElasticSearch index? ")) except ValueError: pass while replicas < 0: try: replicas = int(input("How many replicas for each shard? ")) except ValueError: pass print("Okay, I got all I need, setting up Pony Mail...") def createIndex(): # Check if index already exists if es.indices.exists(dbname): if args.soe: print("ElasticSearch index '%s' already exists and SOE set, exiting quietly" % dbname) sys.exit(0) else: print("Error: ElasticSearch index '%s' already exists!" % dbname) sys.exit(-1) print("Creating index " + dbname) settings = { "number_of_shards" : shards, "number_of_replicas" : replicas } mappings = { "mbox" : { "properties" : { "@import_timestamp" : { "type" : "date", "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" }, "attachments" : { "properties" : { "content_type" : { "type" : "string", "index" : "not_analyzed" }, "filename" : { "type" : "string", "index" : "not_analyzed" }, "hash" : { "type" : "string", "index" : "not_analyzed" }, "size" : { "type" : "long" } } }, "body" : { "type" : "string" }, "cc": { "type": "string" }, "date" : { "type" : "date", "store" : True, "format" : "yyyy/MM/dd HH:mm:ss", "index" : "not_analyzed" }, "epoch" : { # number of seconds since the epoch "type" : "long", "index" : "not_analyzed" }, "from" : { "type" : "string" }, "from_raw" : { "type" : "string", "index" : "not_analyzed" }, "in-reply-to" : { "type" : "string", "index" : "not_analyzed" }, "list" : { "type" : "string" }, "list_raw" : { "type" : "string", "index" : "not_analyzed" }, "message-id" : { "type" : "string", "index" : "not_analyzed" }, "mid" : { "type" : "string" }, "private" : { "type" : "boolean" }, "references" : { "type" : "string" }, "subject" : { "type" : "string", "fielddata": True # dropped later if DB_MAJOR==2 }, "to" : { "type" : "string" } } }, "attachment" : { "properties" : { "source" : { "type" : "binary" } } }, "mbox_source" : { "_all": { "enabled": False # this doc type is not searchable }, "properties" : { "source" : { "type" : "binary" }, "message-id" : { "type" : "string", "index" : "not_analyzed" }, "mid" : { "type" : "string" } } }, "mailinglists" : { "_all": { "enabled": False # this doc type is not searchable }, "properties" : { "description" : { "type" : "string", "index" : "not_analyzed" }, "list" : { "type" : "string", # "index" : "not_analyzed" }, "name" : { "type" : "string", "index" : "not_analyzed" } } }, "account" : { "_all": { "enabled": False # this doc type is not searchable }, "properties" : { "cid" : { "type" : "string", "index" : "not_analyzed" }, "credentials" : { "properties" : { "altemail" : { "type" : "object" }, "email" : { "type" : "string", "index" : "not_analyzed" }, "fullname" : { "type" : "string", "index" : "not_analyzed" }, "uid" : { "type" : "string", "index" : "not_analyzed" } } }, "internal" : { "properties" : { "cookie" : { "type" : "string", "index" : "not_analyzed" }, "ip" : { "type" : "string", "index" : "not_analyzed" }, "oauth_used" : { "type" : "string", "index" : "not_analyzed" } } }, "request_id" : { "type" : "string", "index" : "not_analyzed" } } }, "notifications" : { "_all": { "enabled": False # this doc type is not searchable }, "properties" : { "date" : { "type" : "date", "store" : True, "format" : "yyyy/MM/dd HH:mm:ss" }, "epoch" : { "type" : "long" }, "from" : { "type" : "string", # "index" : "not_analyzed" }, "in-reply-to" : { "type" : "string", "index" : "not_analyzed" }, "list" : { "type" : "string", # "index" : "not_analyzed" }, "message-id" : { "type" : "string", "index" : "not_analyzed" }, "mid" : { "type" : "string", # "index" : "not_analyzed" }, "private" : { "type" : "boolean" }, "recipient" : { "type" : "string", "index" : "not_analyzed" }, "seen" : { "type" : "long" }, "subject" : { "type" : "string", "fielddata": True # dropped later if DB_MAJOR==2 # "index" : "not_analyzed" }, "to" : { "type" : "string", # "index" : "not_analyzed" }, "type" : { "type" : "string", "index" : "not_analyzed" } } } } if DB_MAJOR == 2: # ES 2 handles fielddata differently del mappings['mbox']['properties']['subject']['fielddata'] del mappings['notifications']['properties']['subject']['fielddata'] res = es.indices.create(index = dbname, body = { "mappings" : mappings, "settings": settings } ) print("Index created! %s " % res) # we need to connect to database to determine the engine version es = Elasticsearch([ { 'host': hostname, 'port': port, 'use_ssl': False, 'url_prefix': urlPrefix }], max_retries=5, retry_on_timeout=True ) # elasticsearch logs lots of warnings on retries/connection failure logging.getLogger("elasticsearch").setLevel(logging.ERROR) try: DB_VERSION=es.info()['version']['number'] except ES_ConnectionError: print("WARNING: Connection error: could not determine the engine version.") DB_VERSION='0.0.0' DB_MAJOR=int(DB_VERSION.split('.')[0]) print("Versions: library %d (%s), engine %d (%s)" % (ES_MAJOR, '.'.join(map(str,ES_VERSION)) , DB_MAJOR, DB_VERSION)) if not DB_MAJOR == ES_MAJOR: print("WARNING: library version does not agree with engine version!") if DB_MAJOR == 0: # not known if args.noi: # allow setup to be used without engine running print("Could not determine the engine version. Assume it is the same as the library version.") DB_MAJOR = ES_MAJOR else: # if we cannot connect to get the version, we cannot create the index later print("Could not connect to the engine. Fatal.") sys.exit(1) if not args.noi: try: createIndex() except ElasticsearchException as e: print("Index creation failed: %s" % e) sys.exit(1) ponymail_cfg = 'ponymail.cfg' if not args.clobber and os.path.exists(ponymail_cfg): print("%s exists and clobber is not set" % ponymail_cfg) ponymail_cfg = 'ponymail.cfg.tmp' print("Writing importer config (%s)" % ponymail_cfg) with open(ponymail_cfg, "w") as f: f.write(""" ############################################################### # Pony Mail Configuration file # Main ES configuration [elasticsearch] hostname: %s dbname: %s port: %u ssl: false #uri: url_prefix #user: username #password: password #%s #backup: database name [archiver] generator: %s [debug] #cropout: string to crop from list-id ############################################################### """ % (hostname, dbname, port, 'wait: active shard count' if DB_MAJOR == 5 else 'write: consistency level (default quorum)', genname)) config_path = "../site/api/lib" config_file = "config.lua" if not args.clobber and os.path.exists(os.path.join(config_path,config_file)): print("%s exists and clobber is not set" % config_file) config_file = "config.lua.tmp" print("mod_lua configuration (%s)" % config_file) with open(os.path.join(config_path,config_file), "w") as f: f.write(""" local config = { es_url = "http://%s:%u/%s/", mailserver = "%s", -- mailport = 1025, -- override the default port (25) accepted_domains = "%s", wordcloud = %s, email_footer = nil, -- see the docs for how to set this up. full_headers = false, maxResults = 5000, -- max emails to return in one go. Might need to be bumped for large lists -- stats_maxBody = 200, -- max size of body snippet returned by stats.lua -- stats_wordExclude = ".|..|...", -- patterns to exclude from word cloud generated by stats.lua admin_oauth = {}, -- list of domains that may do administrative oauth (private list access) -- add 'www.googleapis.com' to the list for google oauth to decide, for instance. oauth_fields = { -- used for specifying individual oauth handling parameters. -- for example: -- internal = { -- email = 'CAS-EMAIL', -- name = 'CAS-NAME', -- uid = 'REMOTE-USER', -- env = 'subprocess' -- use environment vars instead of request headers -- } }, -- allow_insecure_cookie = true, -- override the default (false) - only use for test installations -- no_association = {}, -- domains that are not allowed for email association -- listsDisplay = 'regex', -- if defined, hide list names that don't match the regex -- debug = false, -- whether to return debug information antispam = true -- Whether or not to add anti-spam measures aimed at anonymous users. } return config """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false")) print("Copying sample JS config to config.js (if needed)...") if not os.path.exists("../site/js/config.js") and os.path.exists("../site/js/config.js.sample"): shutil.copy("../site/js/config.js.sample", "../site/js/config.js") print("All done, Pony Mail should...work now :)") print("If you are using an external mail inbound server, \nmake sure to copy archiver.py and ponymail.cfg to it")