def fetch_icons()

in make_manifest.py [0:0]

• 32 lines of code
• 14 McCabe index (conditional complexity)

def fetch_icons(url, user_agent=IPHONE_UA):
    logging.info(f'Fetching icons for {url}')
    icons = []
    browser = RoboBrowser(user_agent=user_agent, parser='html.parser')
    try:
        browser.open(url, timeout=60)
        for link in browser.select(LINK_SELECTOR):
            icon = link.attrs
            icon_url = icon['href']
            if icon_url.startswith('data:'):
                continue
            if not icon_url.startswith('http') and not icon_url.startswith('//'):
                icon['href'] = urljoin(browser.url, icon_url)
            icons.append(icon)
        for meta in browser.select(META_SELECTOR):
            icon = meta.attrs
            icon_url = icon['content']
            if icon_url.startswith('data:'):
                continue
            if not icon_url.startswith('http') and not icon_url.startswith('//'):
                icon['href'] = urljoin(browser.url, icon_url)
            else:
                icon['href'] = icon_url
            icons.append(icon)
    except Exception as e:
        logging.info(f'Exception: "{str(e)}" while parsing icon urls from document')
        pass

    # If the document doesn't specify favicon via rel attribute of link tag then check
    # if "favicon.ico" file is present in the root of the domain as some domains keep
    # favicon in their root without specifying them in the document.
    # Add the icon url if this is the case.
    if len(icons) == 0:
        default_favicon_url = f"{url}/favicon.ico"
        if is_url_reachable(default_favicon_url):
            icons.append({"href": default_favicon_url})

    return icons