crashclouseau/buildhub.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import asyncio
from copy import deepcopy
from functools import partial
import json
import requests
import six
import time
from . import utils
from .logger import logger


URL = "https://buildhub.moz.tools/api/search"
PRODS = {"Firefox": "firefox", "FennecAndroid": "fennec", "Thunderbird": "thunderbird"}
RPRODS = {v: k for k, v in PRODS.items()}

# regexp matching the correct version formats for elastic search query
VERSION_PATS = {
    "nightly": r'[0-9]+".0a1"',
    "beta": r'[0-9]+".0b"[0-9]+',
    "release": r"[0-9]+\.[0-9]+(\.[0-9]+)?",
}


def make_request(params, sleep, retry, callback):
    """Query Buildhub"""
    params = json.dumps(params)

    for _ in range(retry):
        r = requests.post(URL, data=params)
        if "Backoff" in r.headers:
            time.sleep(sleep)
        else:
            try:
                return callback(r.json())
            except BaseException as e:
                logger.error(
                    "Buildhub query failed with parameters: {}.".format(params)
                )
                logger.error(e, exc_info=True)
                return None

    logger.error("Too many attempts in buildhub.make_request (retry={})".format(retry))

    return None


def get(
    min_buildid, channel, prods=["firefox", "fennec", "thunderbird"], max_buildid=None
):
    """Get all builds info for buildids >= min_build"""
    if isinstance(prods, six.string_types):
        prods = [prods]
    prods = [PRODS.get(x, x) for x in prods]
    r = {}
    if min_buildid:
        r["gte"] = utils.get_buildid(min_buildid)
    if max_buildid:
        r["lte"] = utils.get_buildid(max_buildid)

    data = {
        "aggs": {
            "products": {
                "terms": {"field": "source.product", "size": len(prods)},
                "aggs": {
                    "channels": {
                        "terms": {"field": "target.channel", "size": 1},
                        "aggs": {
                            "buildids": {
                                "terms": {"field": "build.id", "size": 1000},
                                "aggs": {
                                    "revisions": {
                                        "terms": {"field": "source.revision", "size": 1}
                                    },
                                    "versions": {
                                        "terms": {"field": "target.version", "size": 1}
                                    },
                                },
                            }
                        },
                    }
                },
            }
        },
        "query": {
            "bool": {
                "filter": [
                    {"term": {"target.channel": channel}},
                    {"terms": {"source.product": prods}},
                    {"range": {"build.id": r}},
                    {"regexp": {"target.version": VERSION_PATS.get(channel, "*")}},
                ]
            }
        },
        "size": 0,
    }

    def get_info(data):
        res = {}
        aggs = data["aggregations"]
        for product in aggs["products"]["buckets"]:
            prod = product["key"]
            prod = RPRODS.get(prod, prod)
            if prod in res:
                res_p = res[prod]
            else:
                res[prod] = res_p = {}
            for channel in product["channels"]["buckets"]:
                chan = channel["key"]
                if chan in res_p:
                    res_pc = res_p[chan]
                else:
                    res_p[chan] = res_pc = {}

                for buildid in channel["buildids"]["buckets"]:
                    bid = utils.get_build_date(buildid["key"])
                    rev = buildid["revisions"]["buckets"][0]["key"]
                    version = buildid["versions"]["buckets"][0]["key"]
                    res_pc[bid] = {"revision": utils.short_rev(rev), "version": version}
        return res

    return make_request(data, 1, 100, get_info)


def get_rev_from(buildid, channel, product):
    """Get the revision for a given build"""
    buildid = utils.get_buildid(buildid)
    product = PRODS.get(product, product)
    data = {
        "aggs": {"revisions": {"terms": {"field": "source.revision", "size": 1}}},
        "query": {
            "bool": {
                "filter": [
                    {"term": {"target.channel": channel}},
                    {"term": {"source.product": product}},
                    {"term": {"build.id": buildid}},
                ]
            }
        },
        "size": 0,
    }

    def cb(data):
        return utils.short_rev(data["aggregations"]["revisions"]["buckets"][0]["key"])

    return make_request(data, 0.1, 100, cb)


def get_two_last(buildid, channel, product):
    """Get the two last build (including the one from buildid)"""
    buildid = utils.get_buildid(buildid)
    product = PRODS.get(product, product)
    data = {
        "aggs": {
            "buildids": {
                "terms": {"field": "build.id", "size": 2, "order": {"_term": "desc"}},
                "aggs": {
                    "revisions": {"terms": {"field": "source.revision", "size": 1}},
                    "versions": {"terms": {"field": "target.version", "size": 1}},
                },
            }
        },
        "query": {
            "bool": {
                "filter": [
                    {"term": {"target.channel": channel}},
                    {"term": {"source.product": product}},
                    {"range": {"build.id": {"lte": buildid}}},
                    {"regexp": {"target.version": VERSION_PATS.get(channel, "*")}},
                ]
            }
        },
        "size": 0,
    }

    def get_info(data):
        bids = []
        for i in data["aggregations"]["buildids"]["buckets"]:
            bid = i["key"]
            revision = utils.short_rev(i["revisions"]["buckets"][0]["key"])
            version = i["versions"]["buckets"][0]["key"]
            bids.append({"buildid": bid, "revision": revision, "version": version})

        if bids[0]["buildid"] != buildid:
            return None

        x = bids[0]
        bids[0] = bids[1]
        bids[1] = x

        return bids

    return make_request(data, 0.1, 100, get_info)


async def get_enclosing_builds_helper(pushdate, channel, product):
    # TODO: we must handle the case where the timezone of buildid was not utc
    # check with jlorenzo when the changed has been made
    buildid = utils.get_buildid(pushdate)
    product = PRODS.get(product, product)
    lt_data = {
        "aggs": {
            "buildids": {
                "terms": {"field": "build.id", "size": 1, "order": {"_term": "desc"}},
                "aggs": {
                    "revisions": {"terms": {"field": "source.revision", "size": 1}},
                    "versions": {"terms": {"field": "target.version", "size": 1}},
                },
            }
        },
        "query": {
            "bool": {
                "filter": [
                    {"term": {"target.channel": channel}},
                    {"term": {"source.product": product}},
                    {"range": {"build.id": {"lt": buildid}}},
                    {"regexp": {"target.version": VERSION_PATS.get(channel, "*")}},
                ]
            }
        },
        "size": 0,
    }

    gte_data = deepcopy(lt_data)
    gte_data["aggs"]["buildids"]["terms"]["order"]["_term"] = "asc"
    gte_data["query"]["bool"]["filter"][2]["range"]["build.id"] = {"gte": buildid}
    data = [lt_data, gte_data]

    def get_info(data):
        data = data["aggregations"]["buildids"]["buckets"]
        if len(data) == 0:
            return None
        data = data[0]
        bid = data["key"]
        revision = utils.short_rev(data["revisions"]["buckets"][0]["key"])
        version = data["versions"]["buckets"][0]["key"]
        return {"buildid": bid, "revision": revision, "version": version}

    loop = asyncio.get_event_loop()
    fs = []
    for d in data:
        fs.append(
            loop.run_in_executor(None, partial(make_request, d, 0.1, 100, get_info))
        )
    res = []
    for f in fs:
        res.append(await f)

    return res


def get_enclosing_builds(pushdate, channel, product):
    """Get the build before and the one after the given pushdate"""
    loop = asyncio.get_event_loop()
    return loop.run_until_complete(
        get_enclosing_builds_helper(pushdate, channel, product)
    )