public void configure()

in core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java [87:154]
51 lines of code
5 McCabe index (conditional complexity)

    public void configure(final Config conf) {

        super.configure(conf);

        // allow up to 200 connections or same as the number of threads used for
        // fetching
        int maxFetchThreads = ConfUtils.getInt(conf, "fetcher.threads.number", 200);
        CONNECTION_MANAGER.setMaxTotal(maxFetchThreads);
        int maxPerRoute = ConfUtils.getInt(conf, "fetcher.threads.per.queue", 1);
        if (maxPerRoute < 20) {
            maxPerRoute = 20;
        }
        CONNECTION_MANAGER.setDefaultMaxPerRoute(maxPerRoute);

        globalMaxContent = ConfUtils.getInt(conf, "http.content.limit", -1);

        String userAgent = getAgentString(conf);

        Collection<BasicHeader> defaultHeaders = new LinkedList<>();

        String accept = ConfUtils.getString(conf, "http.accept");
        if (StringUtils.isNotBlank(accept)) {
            defaultHeaders.add(new BasicHeader("Accept", accept));
        }

        customHeaders.forEach(
                h -> {
                    defaultHeaders.add(new BasicHeader(h.getKey(), h.getValue()));
                });

        String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null);

        // use a basic auth?
        if (StringUtils.isNotBlank(basicAuthUser)) {
            String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", "");
            String encoding =
                    Base64.getEncoder()
                            .encodeToString(
                                    (basicAuthUser + ":" + basicAuthPass)
                                            .getBytes(StandardCharsets.UTF_8));
            defaultHeaders.add(new BasicHeader("Authorization", "Basic " + encoding));
        }

        String acceptLanguage = ConfUtils.getString(conf, "http.accept.language");
        if (StringUtils.isNotBlank(acceptLanguage)) {
            defaultHeaders.add(new BasicHeader("Accept-Language", acceptLanguage));
        }

        builder =
                HttpClients.custom()
                        .setUserAgent(userAgent)
                        .setDefaultHeaders(defaultHeaders)
                        .setConnectionManager(CONNECTION_MANAGER)
                        .setConnectionManagerShared(true)
                        .disableRedirectHandling()
                        .disableAutomaticRetries();

        int timeout = ConfUtils.getInt(conf, "http.timeout", 10000);

        requestConfigBuilder =
                RequestConfig.custom()
                        .setSocketTimeout(timeout)
                        .setConnectTimeout(timeout)
                        .setConnectionRequestTimeout(timeout)
                        .setCookieSpec(CookieSpecs.STANDARD);

        requestConfig = requestConfigBuilder.build();
    }