apps/mountebank-mock/mountebank-source/src/models/filesystemBackedImpostersRepository.js

'use strict';

const fsExtra = require('fs-extra'),
    prometheus = require('prom-client'),
    properLockFile = require('proper-lockfile'),
    pathModule = require('path'),
    helpers = require('../util/helpers.js'),
    errors = require('../util/errors.js');

/**
 * An abstraction for loading imposters from the filesystem
 * The root of the database is provided on the command line as --datadir
 * The file layout looks like this:
 *
 * /{datadir}
 *   /3000
 *     /imposter.json
 *       {
 *         "protocol": "http",
 *         "port": 3000,
 *         "stubs: [{
 *           "predicates": [{ "equals": { "path": "/" } }],
 *           "meta": {
 *             "dir": "stubs/{epoch-pid-counter}"
 *           }
 *         }]
 *       }
 *
 *     /stubs
 *       /{epoch-pid-counter}
 *         /meta.json
 *           {
 *             "responseFiles": ["responses/{epoch-pid-counter}.json"],
 *               // An array of indexes into responseFiles which handle repeat behavior
 *             "orderWithRepeats": [0],
 *               // The next index into orderWithRepeats; incremented with each call to nextResponse()
 *             "nextIndex": 0
 *           }
 *
 *         /responses
 *           /{epoch-pid-counter}.json
 *             {
 *               "is": { "body": "Hello, world!" }
 *             }
 *
 *         /matches
 *             /{epoch-pid-counter}.json
 *                 { ... }
 *
 *     /requests
 *       /{epoch-pid-counter}.json
 *         { ... }
 *
 * This structure is designed to minimize the amount of file locking and to maximize parallelism and throughput.
 *
 * The imposters.json file needs to be locked during imposter-level activities (e.g. adding a stub).
 * Readers do not lock; they just get the data at the time they read it. Since this file doesn't
 * contain transient state, there's no harm from a stale read. Writes (which happen for
 * stub changing operations, either for proxy response recording or stub API calls) grab a file lock
 * for both the read and the write. Writes to this file should be infrequent, except perhaps during
 * proxy recording. Newly added stubs may change the index of existing stubs in the stubs array, but
 * will never change the stub meta.dir, so it is always safe to hang on to it for subsequent operations.
 *
 * The stub meta.json needs to be locked to add responses or trigger the next response, but is
 * separated from the imposter.json so we can have responses from multiple stubs in parallel with no
 * lock conflict. Again, readers (e.g. to generate imposter JSON) do not need a lock because the responseFiles
 * array is mostly read-only, and even when it's not (adding responses during proxyAlways recording), there's
 * no harm from a stale read. Writers (usually generating the next response for a stub) grab a lock for the
 * read and the write. This should be the most common write across files, which is why the meta.json file
 * is small.
 *
 * In both cases where a file needs to be locked, an exponential backoff retry strategy is used. Inconsistent
 * reads of partially written files (which can happen by default with the system calls - fs.writeFile is not
 * atomic) are avoided by writing first to a temp file (during which time reads can happen to the original file)
 * and then renaming to the original file.
 *
 * New directories and filenames use a timestamp-based filename to allow creating them without synchronizing
 * with a read. Since multiple files (esp. requests) can be added during the same millisecond, a pid and counter
 * is tacked on to the filename to improve uniqueness. It doesn't provide the * ironclad guarantee a GUID
 * does -- two different processes on two different machines could in theory have the same pid and create files
 * during the same timestamp with the same counter, but the odds of that happening * are so small that it's not
 * worth giving up the easy time-based sortability based on filenames alone.
 *
 * Keeping all imposter information under a directory (instead of having metadata outside the directory)
 * allows us to remove the imposter by simply removing the directory.
 *
 * There are some extra checks on filesystem operations (atomicWriteFile) due to antivirus software, solar flares,
 * gremlins, etc. graceful-fs solves some of these, but apparently not all.
 *
 * @module
 */

const
    metrics = {
        lockAcquireDuration: new prometheus.Histogram({
            name: 'mb_lock_acquire_duration_seconds',
            help: 'Time it takes to acquire a file lock',
            buckets: [0.1, 0.2, 0.5, 1, 3, 5, 10, 30],
            labelNames: ['caller']
        }),
        lockHoldDuration: new prometheus.Histogram({
            name: 'mb_lock_hold_duration_seconds',
            help: 'Time a file lock is held',
            buckets: [0.1, 0.2, 0.5, 1, 2],
            labelNames: ['caller']
        }),
        lockErrors: new prometheus.Counter({
            name: 'mb_lock_errors_total',
            help: 'Number of lock errors',
            labelNames: ['caller', 'code']
        })
    };

/**
 * Creates the repository
 * @param {Object} config - The database configuration
 * @param {String} config.datadir - The database directory
 * @param {Object} logger - The logger
 * @returns {Object}
 */
function create (config, logger) {
    const imposterFns = {};
    let counter = 0,
        locks = 0;

    async function ensureDir (filepath) {
        const dir = pathModule.dirname(filepath);

        await fsExtra.ensureDir(dir);
    }

    async function ensureFile (filepath) {
        fsExtra.close(await fsExtra.open(filepath, 'as'));
    }

    async function writeFile (filepath, obj) {
        await ensureDir(filepath);
        await ensureFile(filepath);
        await fsExtra.writeFile(filepath, JSON.stringify(obj, null, 2), {
            flag: 'rs+'
        });
    }

    function tryParse (maybeJSON, filepath) {
        try {
            return JSON.parse(maybeJSON);
        }
        catch (parseErr) {
            logger.error(`Corrupted database: invalid JSON for ${filepath}`);
            throw errors.DatabaseError(`invalid JSON in ${filepath}`, { details: parseErr.message });
        }
    }

    async function readFile (filepath, defaultContents) {
        try {
            const data = await fsExtra.readFile(filepath, 'utf8');
            return tryParse(data, filepath);
        }
        catch (err) {
            if (err.code === 'ENOENT') {
                if (typeof defaultContents === 'undefined') {
                    logger.error(`Corrupted database: missing file ${filepath}`);
                    throw errors.DatabaseError('file not found', { details: err.message });
                }
                else {
                    return defaultContents;
                }
            }
            else {
                throw err;
            }
        }
    }

    function delay (duration) {
        return new Promise(resolve => {
            setTimeout(resolve, duration);
        });
    }

    async function atomicWriteFile (filepath, obj, attempts = 1) {
        const tmpfile = filepath + '.tmp';

        try {
            await writeFile(tmpfile, obj);
            await fsExtra.rename(tmpfile, filepath);
        }
        catch (err) {
            if (err.code === 'ENOENT' && attempts < 15) {
                logger.debug(`Attempt ${attempts} failed with ENOENT error on atomic write of ${filepath}. Retrying...`);
                await delay(10);
                await atomicWriteFile(filepath, obj, attempts + 1);
            }
            else {
                throw err;
            }
        }
    }

    async function lockFile (filepath) {
        const options = {
            realpath: false,
            retries: {
                retries: 20,
                minTimeout: 10,
                maxTimeout: 5000,
                randomize: true,
                factor: 1.5
            },
            stale: 30000
        };

        // with realpath = false, the file doesn't have to exist, but the directory does
        await ensureDir(filepath);
        return properLockFile.lock(filepath, options);
    }

    async function readAndWriteFile (filepath, caller, transformer, defaultContents) {
        const currentLockId = locks,
            observeLockAcquireDuration = metrics.lockAcquireDuration.startTimer({ caller });

        locks += 1;

        try {
            const release = await lockFile(filepath),
                acquireLockSeconds = observeLockAcquireDuration(),
                observeLockHoldDuration = metrics.lockHoldDuration.startTimer({ caller });

            logger.debug(`Acquired file lock on ${filepath} for ${caller}-${currentLockId} after ${acquireLockSeconds}s`);

            const original = await readFile(filepath, defaultContents),
                transformed = await transformer(original);

            await atomicWriteFile(filepath, transformed);
            await release();

            const lockHeld = observeLockHoldDuration();
            logger.debug(`Released file lock on ${filepath} for ${caller}-${currentLockId} after ${lockHeld}s`);
        }
        catch (err) {
            // Ignore lock already released errors
            if (err.code !== 'ERELEASED') {
                metrics.lockErrors.inc({ caller, code: err.code });
                properLockFile.unlock(filepath, { realpath: false }).catch(() => { /* ignore */ });
                throw err;
            }
        }
    }

    async function remove (path) {
        await fsExtra.remove(path);
    }

    function filenameFor (timestamp) {
        const epoch = timestamp.valueOf();
        counter += 1;
        return `${epoch}-${process.pid}-${counter}`;
    }

    function partsFrom (filename) {
        // format {epoch}-{pid}-{counter}
        const pattern = /^(\d+)-(\d+)-(\d+)\.json$/,
            parts = pattern.exec(filename);
        return {
            epoch: Number(parts[1]),
            pid: Number(parts[2]),
            counter: Number(parts[3])
        };
    }

    function timeSorter (first, second) {
        // format {epoch}-{pid}-{counter}
        // sort by epoch first, then pid, then counter to guarantee determinism for
        // files added during the same millisecond.
        const firstParts = partsFrom(first),
            secondParts = partsFrom(second);
        let result = firstParts.epoch - secondParts.epoch;
        if (result === 0) {
            result = firstParts.pid - secondParts.pid;
        }
        if (result === 0) {
            result = firstParts.counter - secondParts.counter;
        }
        return result;
    }

    function readdir (path) {
        return new Promise((resolve, reject) => {
            fsExtra.readdir(path, (err, files) => {
                if (err && err.code === 'ENOENT') {
                    // Nothing saved yet
                    resolve([]);
                }
                else if (err) {
                    reject(err);
                }
                else {
                    resolve(files);
                }
            });
        });
    }

    async function loadAllInDir (path) {
        const files = await readdir(path),
            reads = files
                .filter(file => file.indexOf('.json') > 0)
                .sort(timeSorter)
                .map(file => readFile(`${path}/${file}`));

        return Promise.all(reads);
    }

    function repeatsFor (response) {
        return response.repeat || 1;
    }

    async function saveStubMetaAndResponses (stub, baseDir) {
        const stubDefinition = {
                meta: { dir: `stubs/${filenameFor(new Date())}` }
            },
            meta = {
                responseFiles: [],
                orderWithRepeats: [],
                nextIndex: 0
            },
            responses = stub.responses || [],
            writes = [];

        if (stub.predicates) {
            stubDefinition.predicates = stub.predicates;
        }

        for (let i = 0; i < responses.length; i += 1) {
            const responseFile = `responses/${filenameFor(new Date())}.json`;
            meta.responseFiles.push(responseFile);

            for (let repeats = 0; repeats < repeatsFor(responses[i]); repeats += 1) {
                meta.orderWithRepeats.push(i);
            }

            writes.push(writeFile(`${baseDir}/${stubDefinition.meta.dir}/${responseFile}`, responses[i]));
        }

        writes.push(writeFile(`${baseDir}/${stubDefinition.meta.dir}/meta.json`, meta));
        await Promise.all(writes);
        return stubDefinition;
    }

    function stubRepository (baseDir) {
        const imposterFile = `${baseDir}/imposter.json`;

        function metaPath (stubDir) {
            return `${baseDir}/${stubDir}/meta.json`;
        }

        function responsePath (stubDir, responseFile) {
            return `${baseDir}/${stubDir}/${responseFile}`;
        }

        function requestPath (request) {
            return `${baseDir}/requests/${filenameFor(Date.parse(request.timestamp))}.json`;
        }

        function matchPath (stubDir, match) {
            return `${baseDir}/${stubDir}/matches/${filenameFor(Date.parse(match.timestamp))}.json`;
        }

        function readHeader () {
            return readFile(imposterFile, { stubs: [] });
        }

        function readAndWriteHeader (caller, transformer) {
            return readAndWriteFile(imposterFile, caller, transformer, { stubs: [] });
        }

        function wrap (stub) {
            const cloned = helpers.clone(stub || {}),
                stubDir = stub ? stub.meta.dir : '';

            if (typeof stub === 'undefined') {
                return {
                    addResponse: () => Promise.resolve(),
                    nextResponse: () => Promise.resolve({
                        is: {},
                        stubIndex: () => Promise.resolve(0)
                    }),
                    recordMatch: () => Promise.resolve()
                };
            }

            delete cloned.meta;

            /**
             * Adds a response to the stub
             * @memberOf module:models/filesystemBackedImpostersRepository#
             * @param {Object} response - the new response
             * @returns {Object} - the promise
             */
            cloned.addResponse = async response => {
                let responseFile;
                await readAndWriteFile(metaPath(stubDir), 'addResponse', async meta => {
                    const responseIndex = meta.responseFiles.length;
                    responseFile = `responses/${filenameFor(new Date())}.json`;

                    meta.responseFiles.push(responseFile);
                    for (let repeats = 0; repeats < repeatsFor(response); repeats += 1) {
                        meta.orderWithRepeats.push(responseIndex);
                    }

                    await writeFile(responsePath(stubDir, responseFile), response);
                    return meta;
                });
            };

            async function stubIndex () {
                const header = await readHeader();
                for (let i = 0; i < header.stubs.length; i += 1) {
                    if (header.stubs[i].meta.dir === stubDir) {
                        return i;
                    }
                }
                return 0;
            }

            function createResponse (responseConfig) {
                const result = helpers.clone(responseConfig || { is: {} });
                result.stubIndex = stubIndex;
                return result;
            }

            /**
             * Returns the next response for the stub, taking into consideration repeat behavior and cycling back the beginning
             * @memberOf module:models/filesystemBackedImpostersRepository#
             * @returns {Object} - the promise
             */
            cloned.nextResponse = async () => {
                let responseFile;
                await readAndWriteFile(metaPath(stubDir), 'nextResponse', async meta => {
                    const maxIndex = meta.orderWithRepeats.length,
                        responseIndex = meta.orderWithRepeats[meta.nextIndex % maxIndex];

                    responseFile = meta.responseFiles[responseIndex];

                    meta.nextIndex = (meta.nextIndex + 1) % maxIndex;
                    return meta;
                });

                // No need to read the response file while the lock is held
                const responseConfig = await readFile(responsePath(stubDir, responseFile));
                return createResponse(responseConfig);
            };

            /**
             * Records a match for debugging purposes
             * @memberOf module:models/filesystemBackedImpostersRepository#
             * @param {Object} request - the request
             * @param {Object} response - the response
             * @param {Object} responseConfig - the config that generated the response
             * @param {Number} processingTime - the time to match the predicate and generate the full response
             * @returns {Object} - the promise
             */
            cloned.recordMatch = async (request, response, responseConfig, processingTime) => {
                const match = {
                    timestamp: new Date().toJSON(),
                    request,
                    response,
                    responseConfig,
                    processingTime
                };
                await writeFile(matchPath(stubDir, match), match);
            };

            return cloned;
        }

        /**
         * Returns the number of stubs for the imposter
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @returns {Object} - the promise
         */
        async function count () {
            const imposter = await readHeader();
            return imposter.stubs.length;
        }

        /**
         * Returns the first stub whose predicates matches the filter
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Function} filter - the filter function
         * @param {Number} startIndex - the index to to start searching
         * @returns {Object} - the promise
         */
        async function first (filter, startIndex = 0) {
            const header = await readHeader();

            for (let i = startIndex; i < header.stubs.length; i += 1) {
                if (filter(header.stubs[i].predicates || [])) {
                    return { success: true, stub: wrap(header.stubs[i]) };
                }
            }
            return { success: false, stub: wrap() };
        }

        /**
         * Adds a new stub to imposter
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Object} stub - the stub to add
         * @returns {Object} - the promise
         */
        async function add (stub) { // eslint-disable-line no-shadow
            const stubDefinition = await saveStubMetaAndResponses(stub, baseDir);

            await readAndWriteHeader('addStub', async header => {
                header.stubs.push(stubDefinition);
                return header;
            });
        }

        /**
         * Inserts a new stub at the given index
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Object} stub - the stub to add
         * @param {Number} index - the index to insert the new stub at
         * @returns {Object} - the promise
         */
        async function insertAtIndex (stub, index) {
            const stubDefinition = await saveStubMetaAndResponses(stub, baseDir);

            await readAndWriteHeader('insertStubAtIndex', async header => {
                header.stubs.splice(index, 0, stubDefinition);
                return header;
            });
        }

        /**
         * Deletes the stub at the given index
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Number} index - the index of the stub to delete
         * @returns {Object} - the promise
         */
        async function deleteAtIndex (index) {
            let stubDir;

            await readAndWriteHeader('deleteStubAtIndex', async header => {
                if (typeof header.stubs[index] === 'undefined') {
                    throw errors.MissingResourceError(`no stub at index ${index}`);
                }

                stubDir = header.stubs[index].meta.dir;
                header.stubs.splice(index, 1);
                return header;
            });

            await remove(`${baseDir}/${stubDir}`);
        }

        /**
         * Overwrites all stubs with a new list
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Object} newStubs - the new list of stubs
         * @returns {Object} - the promise
         */
        async function overwriteAll (newStubs) {
            await readAndWriteHeader('overwriteAllStubs', async header => {
                header.stubs = [];
                await remove(`${baseDir}/stubs`);
                return header;
            });

            let addSequence = Promise.resolve();
            newStubs.forEach(stub => {
                addSequence = addSequence.then(() => add(stub));
            });
            await addSequence;
        }

        /**
         * Overwrites the stub at the given index
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Object} stub - the new stub
         * @param {Number} index - the index of the stub to overwrite
         * @returns {Object} - the promise
         */
        async function overwriteAtIndex (stub, index) {
            await deleteAtIndex(index);
            await insertAtIndex(stub, index);
        }

        async function loadResponses (stub) {
            const meta = await readFile(metaPath(stub.meta.dir));
            return Promise.all(meta.responseFiles.map(responseFile =>
                readFile(responsePath(stub.meta.dir, responseFile))));
        }

        async function loadMatches (stub) {
            return loadAllInDir(`${baseDir}/${stub.meta.dir}/matches`);
        }

        /**
         * Returns a JSON-convertible representation
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Object} options - The formatting options
         * @param {Boolean} options.debug - If true, includes debug information
         * @returns {Object} - the promise resolving to the JSON object
         */
        async function toJSON (options = {}) {
            const header = await readHeader(),
                responsePromises = header.stubs.map(loadResponses),
                stubResponses = await Promise.all(responsePromises),
                debugPromises = options.debug ? header.stubs.map(loadMatches) : [],
                matches = await Promise.all(debugPromises);

            header.stubs.forEach((stub, index) => {
                stub.responses = stubResponses[index];
                if (options.debug && matches[index].length > 0) {
                    stub.matches = matches[index];
                }
                delete stub.meta;
            });

            return header.stubs;
        }

        function isRecordedResponse (response) {
            return response.is && typeof response.is._proxyResponseTime === 'number';
        }

        /**
         * Removes the saved proxy responses
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @returns {Object} - Promise
         */
        async function deleteSavedProxyResponses () {
            const allStubs = await toJSON();
            allStubs.forEach(stub => {
                stub.responses = stub.responses.filter(response => !isRecordedResponse(response));
            });

            const nonProxyStubs = allStubs.filter(stub => stub.responses.length > 0);
            return overwriteAll(nonProxyStubs);
        }

        /**
         * Adds a request for the imposter
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @param {Object} request - the request
         * @returns {Object} - the promise
         */
        async function addRequest (request) {
            const recordedRequest = helpers.clone(request);

            recordedRequest.timestamp = new Date().toJSON();
            await writeFile(requestPath(recordedRequest), recordedRequest);
        }

        /**
         * Returns the saved requests for the imposter
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @returns {Object} - the promise resolving to the array of requests
         */
        async function loadRequests () {
            return loadAllInDir(`${baseDir}/requests`);
        }

        /**
         * Deletes the requests directory for an imposter
         * @memberOf module:models/filesystemBackedImpostersRepository#
         * @returns {Object} - Promise
         */
        async function deleteSavedRequests () {
            await remove(`${baseDir}/requests`);
        }

        return {
            count,
            first,
            add,
            insertAtIndex,
            overwriteAll,
            overwriteAtIndex,
            deleteAtIndex,
            toJSON,
            deleteSavedProxyResponses,
            addRequest,
            loadRequests,
            deleteSavedRequests
        };
    }

    function imposterDir (id) {
        return `${config.datadir}/${id}`;
    }

    function headerFile (id) {
        return `${imposterDir(id)}/imposter.json`;
    }

    /**
     * Returns the stubs repository for the imposter
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Number} id - the id of the imposter
     * @returns {Object} - the stubs repository
     */
    function stubsFor (id) {
        return stubRepository(imposterDir(id));
    }

    /**
     * Saves a reference to the imposter so that the functions
     * (which can't be persisted) can be rehydrated to a loaded imposter.
     * This means that any data in the function closures will be held in
     * memory.
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Object} imposter - the imposter
     */
    function addReference (imposter) {
        const id = String(imposter.port);
        imposterFns[id] = {};
        Object.keys(imposter).forEach(key => {
            if (typeof imposter[key] === 'function') {
                imposterFns[id][key] = imposter[key];
            }
        });
    }

    function rehydrate (imposter) {
        const id = String(imposter.port);
        Object.keys(imposterFns[id]).forEach(key => {
            imposter[key] = imposterFns[id][key];
        });
    }

    /**
     * Adds a new imposter
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Object} imposter - the imposter to add
     * @returns {Object} - the promise
     */
    async function add (imposter) {
        const imposterConfig = imposter.creationRequest,
            stubs = imposterConfig.stubs || [],
            saveStubs = stubs.map(stub => saveStubMetaAndResponses(stub, imposterDir(imposter.port))),
            stubDefinitions = await Promise.all(saveStubs);

        delete imposterConfig.requests;
        imposterConfig.port = imposter.port;
        imposterConfig.stubs = stubDefinitions;
        await writeFile(headerFile(imposter.port), imposterConfig);

        addReference(imposter);
        return imposter;
    }

    /**
     * Gets the imposter by id
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Number} id - the id of the imposter (e.g. the port)
     * @returns {Object} - the promise resolving to the imposter
     */
    async function get (id) {
        const header = await readFile(headerFile(id), null);
        if (header === null) {
            return null;
        }

        header.stubs = await stubsFor(id).toJSON();
        rehydrate(header);
        return header;
    }

    /**
     * Gets all imposters
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @returns {Object} - all imposters keyed by port
     */
    async function all () {
        return Promise.all(Object.keys(imposterFns).map(get));
    }

    /**
     * Returns whether an imposter at the given id exists or not
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Number} id - the id (e.g. the port)
     * @returns {boolean}
     */
    async function exists (id) {
        return Object.keys(imposterFns).indexOf(String(id)) >= 0;
    }

    async function shutdown (id) {
        if (typeof imposterFns[String(id)] === 'undefined') {
            return;
        }

        const stop = imposterFns[String(id)].stop;
        delete imposterFns[String(id)];
        if (stop) {
            await stop();
        }
    }

    /**
     * Deletes the imposter at the given id
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Number} id - the id (e.g. the port)
     * @returns {Object} - the deletion promise
     */
    async function del (id) {
        const imposter = await get(id),
            cleanup = [shutdown(id)];

        if (imposter !== null) {
            cleanup.push(remove(imposterDir(id)));
        }

        await Promise.all(cleanup);
        return imposter;
    }

    /**
     * Deletes all imposters; used during testing
     * @memberOf module:models/filesystemBackedImpostersRepository#
     */
    async function stopAll () {
        const promises = Object.keys(imposterFns).map(shutdown);
        await Promise.all(promises);
    }

    /**
     * Deletes all imposters synchronously; used during shutdown
     * @memberOf module:models/filesystemBackedImpostersRepository#
     */
    function stopAllSync () {
        Object.keys(imposterFns).forEach(shutdown);
    }

    /**
     * Deletes all imposters
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @returns {Object} - the deletion promise
     */
    async function deleteAll () {
        const ids = Object.keys(imposterFns),
            dirs = ids.map(imposterDir),
            deleteImposter = ids.map(shutdown).concat(dirs.map(remove));

        // Remove only the directories for imposters we have a reference to
        await Promise.all(deleteImposter);
        const entries = await readdir(config.datadir);
        if (entries.length === 0) {
            await remove(config.datadir);
        }
    }

    async function loadImposterFrom (dir, protocols) {
        const imposterFilename = `${config.datadir}/${dir}/imposter.json`;

        if (!fsExtra.existsSync(imposterFilename)) {
            logger.warn(`Skipping ${dir} during loading; missing imposter.json`);
            return;
        }

        const imposterConfig = JSON.parse(fsExtra.readFileSync(imposterFilename)),
            protocol = protocols[imposterConfig.protocol];

        if (protocol) {
            logger.info(`Loading ${imposterConfig.protocol}:${dir} from datadir`);
            const imposter = await protocol.createImposterFrom(imposterConfig);
            addReference(imposter);
        }
        else {
            logger.error(`Cannot load imposter ${dir}; no protocol loaded for ${config.protocol}`);
        }
    }

    /**
     * Loads all saved imposters at startup
     * @memberOf module:models/filesystemBackedImpostersRepository#
     * @param {Object} protocols - The protocol map, used to instantiate a new instance
     * @returns {Object} - a promise
     */
    async function loadAll (protocols) {
        if (!config.datadir || !fsExtra.existsSync(config.datadir)) {
            return;
        }

        const dirs = fsExtra.readdirSync(config.datadir),
            promises = dirs.map(async dir => loadImposterFrom(dir, protocols));
        await Promise.all(promises);
    }

    return {
        add,
        addReference,
        get,
        all,
        exists,
        del,
        stopAll,
        stopAllSync,
        deleteAll,
        stubsFor,
        loadAll
    };
}

module.exports = { create };