public void configure()

in core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java [135:259]
103 lines of code
15 McCabe index (conditional complexity)

    public void configure(Config conf) {
        super.configure(conf);

        globalMaxContent = ConfUtils.getInt(conf, "http.content.limit", -1);

        final int timeout = ConfUtils.getInt(conf, "http.timeout", 10000);

        this.completionTimeout =
                ConfUtils.getInt(conf, "topology.message.timeout.secs", completionTimeout);

        this.partialContentAsTrimmed =
                ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false);

        builder =
                new OkHttpClient.Builder()
                        .retryOnConnectionFailure(
                                ConfUtils.getBoolean(
                                        conf, "http.retry.on.connection.failure", true))
                        .followRedirects(ConfUtils.getBoolean(conf, "http.allow.redirects", false))
                        .connectTimeout(timeout, TimeUnit.MILLISECONDS)
                        .writeTimeout(timeout, TimeUnit.MILLISECONDS)
                        .readTimeout(timeout, TimeUnit.MILLISECONDS);

        // protocols in order of preference, see
        // https://square.github.io/okhttp/4.x/okhttp/okhttp3/-ok-http-client/-builder/protocols/
        final List<okhttp3.Protocol> protocols = new ArrayList<>();
        for (String pVersion : protocolVersions) {
            switch (pVersion) {
                case "h2":
                    protocols.add(okhttp3.Protocol.HTTP_2);
                    break;
                case "h2c":
                    if (protocolVersions.size() > 1) {
                        LOG.error("h2c ignored, it cannot be combined with any other protocol");
                    } else {
                        protocols.add(okhttp3.Protocol.H2_PRIOR_KNOWLEDGE);
                    }
                    break;
                case "http/1.1":
                    protocols.add(okhttp3.Protocol.HTTP_1_1);
                    break;
                case "http/1.0":
                    LOG.warn("http/1.0 ignored, not supported by okhttp for requests");
                    break;
                default:
                    LOG.error("{}: unknown protocol version", pVersion);
                    break;
            }
        }
        if (protocols.size() > 0) {
            LOG.info("Using protocol versions: {}", protocols);
            builder.protocols(protocols);
        }

        final String userAgent = getAgentString(conf);
        if (StringUtils.isNotBlank(userAgent)) {
            customRequestHeaders.add(new KeyValue("User-Agent", userAgent));
        }

        final String accept = ConfUtils.getString(conf, "http.accept");
        if (StringUtils.isNotBlank(accept)) {
            customRequestHeaders.add(new KeyValue("Accept", accept));
        }

        final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language");
        if (StringUtils.isNotBlank(acceptLanguage)) {
            customRequestHeaders.add(new KeyValue("Accept-Language", acceptLanguage));
        }

        final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null);

        // use a basic auth?
        if (StringUtils.isNotBlank(basicAuthUser)) {
            final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", "");
            final String encoding =
                    Base64.getEncoder()
                            .encodeToString(
                                    (basicAuthUser + ":" + basicAuthPass)
                                            .getBytes(StandardCharsets.UTF_8));
            customRequestHeaders.add(new KeyValue("Authorization", "Basic " + encoding));
        }

        customHeaders.forEach(customRequestHeaders::add);

        if (storeHTTPHeaders) {
            builder.addNetworkInterceptor(new HTTPHeadersInterceptor());
        }

        if (ConfUtils.getBoolean(conf, "http.trust.everything", true)) {
            builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]);
            builder.hostnameVerifier(
                    new HostnameVerifier() {
                        @Override
                        public boolean verify(String hostname, SSLSession session) {
                            return true;
                        }
                    });
        }

        builder.eventListenerFactory(
                new Factory() {
                    @Override
                    public EventListener create(Call call) {
                        return new DNSResolutionListener(DNStimes);
                    }
                });

        // enable support for Brotli compression (Content-Encoding)
        builder.addInterceptor(BrotliInterceptor.INSTANCE);

        final Map<String, Object> connectionPoolConf =
                (Map<String, Object>) conf.get("okhttp.protocol.connection.pool");
        if (connectionPoolConf != null) {
            final int size = ConfUtils.getInt(connectionPoolConf, "max.idle.connections", 5);
            final int time = ConfUtils.getInt(connectionPoolConf, "connection.keep.alive", 300);
            builder.connectionPool(new ConnectionPool(size, time, TimeUnit.SECONDS));
            LOG.info(
                    "Using connection pool with max. {} idle connections "
                            + "and {} sec. connection keep-alive time",
                    size,
                    time);
        }

        client = builder.build();
    }