public void execute()

in x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java [165:857]


    public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
        final boolean inEnrollmentMode = options.has(enrollmentTokenParam);

        // skipping security auto-configuration because node considered as restarting.
        for (Path dataPath : env.dataDirs()) {
            if (Files.isDirectory(dataPath) && false == isDirEmpty(dataPath)) {
                final String msg = "Skipping security auto configuration because it appears that the node is not starting up for the "
                    + "first time. The node might already be part of a cluster and this auto setup utility is designed to configure "
                    + "Security for new clusters only.";
                notifyOfFailure(inEnrollmentMode, terminal, Terminal.Verbosity.VERBOSE, ExitCodes.NOOP, msg);
            }
        }

        // pre-flight checks for the files that are going to be changed
        final Path ymlPath = env.configDir().resolve("elasticsearch.yml");
        // it is odd for the `elasticsearch.yml` file to be missing or not be a regular (the node won't start)
        // but auto configuration should not be concerned with fixing it (by creating the file) and let the node startup fail
        if (false == Files.exists(ymlPath) || false == Files.isRegularFile(ymlPath, LinkOption.NOFOLLOW_LINKS)) {
            final String msg = String.format(
                Locale.ROOT,
                "Skipping security auto configuration because the configuration file [%s] is missing or is not a regular file",
                ymlPath
            );
            notifyOfFailure(inEnrollmentMode, terminal, Terminal.Verbosity.NORMAL, ExitCodes.CONFIG, msg);
        }
        // If the node's yml configuration is not readable, most probably auto-configuration isn't run under the suitable user
        if (false == Files.isReadable(ymlPath)) {
            final String msg = String.format(
                Locale.ROOT,
                "Skipping security auto configuration because the current user does not have permission to read "
                    + " configuration file [%s]",
                ymlPath
            );
            notifyOfFailure(inEnrollmentMode, terminal, Terminal.Verbosity.NORMAL, ExitCodes.NOOP, msg);
        }
        final Path keystorePath = KeyStoreWrapper.keystorePath(env.configDir());
        // Inform that auto-configuration will not run if keystore cannot be read.
        if (Files.exists(keystorePath)
            && (false == Files.isRegularFile(keystorePath, LinkOption.NOFOLLOW_LINKS) || false == Files.isReadable(keystorePath))) {
            final String msg = String.format(
                Locale.ROOT,
                "Skipping security auto configuration because the node keystore file [%s] is not a readable regular file",
                keystorePath
            );
            notifyOfFailure(inEnrollmentMode, terminal, Terminal.Verbosity.NORMAL, ExitCodes.NOOP, msg);
        }

        if (inReconfigureMode) {
            if (false == inEnrollmentMode) {
                throw new UserException(ExitCodes.USAGE, "enrollment-token is a mandatory parameter when reconfiguring the node");
            }
            env = possiblyReconfigureNode(env, terminal, options, processInfo);
        }

        // only perform auto-configuration if the existing configuration is not conflicting (eg Security already enabled)
        // if it is, silently skip auto configuration
        checkExistingConfiguration(env.settings(), inEnrollmentMode, terminal);

        final ZonedDateTime autoConfigDate = ZonedDateTime.now(ZoneOffset.UTC);
        final Path tempGeneratedTlsCertsDir = env.configDir()
            .resolve(String.format(Locale.ROOT, TLS_GENERATED_CERTS_DIR_NAME + ".%d.tmp", autoConfigDate.toInstant().getEpochSecond()));
        try {
            // it is useful to pre-create the sub-config dir in order to check that the config dir is writable and that file owners match
            Files.createDirectory(tempGeneratedTlsCertsDir);
            // set permissions to 750, don't rely on umask, we assume auto configuration preserves ownership so we don't have to
            // grant "group" or "other" permissions
            PosixFileAttributeView view = Files.getFileAttributeView(tempGeneratedTlsCertsDir, PosixFileAttributeView.class);
            if (view != null) {
                view.setPermissions(PosixFilePermissions.fromString("rwxr-x---"));
            }
        } catch (Throwable t) {
            try {
                deleteDirectory(tempGeneratedTlsCertsDir);
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            // the config dir is probably read-only, either because this auto-configuration runs as a different user from the install user,
            // or if the admin explicitly makes configuration immutable (read-only), both of which are reasons to skip auto-configuration
            // this will show a message to the console (the first time the node starts) and auto-configuration is effectively bypassed
            // the message will not be subsequently shown (because auto-configuration doesn't run for node restarts)
            throw new UserException(ExitCodes.CANT_CREATE, "Could not create auto-configuration directory", t);
        }

        // Check that the created auto-config dir has the same owner as the config dir.
        // This is a sort of sanity check.
        // If the node process works OK given the owner of the config dir, it should also tolerate the auto-created config dir,
        // provided that they both have the same owner and permissions.
        final UserPrincipal newFileOwner = Files.getOwner(tempGeneratedTlsCertsDir, LinkOption.NOFOLLOW_LINKS);
        if (false == newFileOwner.equals(Files.getOwner(env.configDir(), LinkOption.NOFOLLOW_LINKS))) {
            // the following is only printed once, if the node starts successfully
            UserException userException = new UserException(
                ExitCodes.CONFIG,
                "Aborting auto configuration because of config dir ownership mismatch. Config dir is owned by "
                    + Files.getOwner(env.configDir(), LinkOption.NOFOLLOW_LINKS).getName()
                    + " but auto-configuration directory would be owned by "
                    + newFileOwner.getName()
            );
            try {
                deleteDirectory(tempGeneratedTlsCertsDir);
            } catch (Exception ex) {
                userException.addSuppressed(ex);
            }
            throw userException;
        }
        final X509Certificate transportCaCert;
        final PrivateKey transportKey;
        final X509Certificate transportCert;
        final PrivateKey httpCaKey;
        final X509Certificate httpCaCert;
        final PrivateKey httpKey;
        final X509Certificate httpCert;
        final List<String> transportAddresses;
        final String cnValue = NODE_NAME_SETTING.exists(env.settings()) ? NODE_NAME_SETTING.get(env.settings()) : System.getenv("HOSTNAME");
        final X500Principal certificatePrincipal = new X500Principal("CN=" + cnValue);
        final X500Principal httpCaPrincipal = new X500Principal(AUTO_CONFIG_HTTP_ALT_DN);
        final X500Principal transportCaPrincipal = new X500Principal(AUTO_CONFIG_TRANSPORT_ALT_DN);

        if (inEnrollmentMode) {
            // this is an enrolling node, get HTTP CA key/certificate and transport layer key/certificate from another node
            final EnrollmentToken enrollmentToken;
            try {
                enrollmentToken = EnrollmentToken.decodeFromString(enrollmentTokenParam.value(options));
            } catch (Exception e) {
                try {
                    deleteDirectory(tempGeneratedTlsCertsDir);
                } catch (Exception ex) {
                    e.addSuppressed(ex);
                }
                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, "");
                terminal.errorPrintln(
                    Terminal.Verbosity.VERBOSE,
                    "Failed to parse enrollment token : " + enrollmentTokenParam.value(options) + " . Error was: " + e.getMessage()
                );
                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, "");
                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, ExceptionsHelper.stackTrace(e));
                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, "");
                throw new UserException(ExitCodes.DATA_ERROR, "Aborting auto configuration. Invalid enrollment token", e);
            }

            final CommandLineHttpClient client = clientFunction.apply(env, enrollmentToken.getFingerprint());

            // We don't wait for cluster health here. If the user has a token, it means that at least the first node has started
            // successfully so we expect the cluster to be healthy already. If not, this is a sign of a problem and we should bail.
            HttpResponse enrollResponse = null;
            URL enrollNodeUrl = null;
            for (String address : enrollmentToken.getBoundAddress()) {
                try {
                    enrollNodeUrl = createURL(new URL("https://" + address), "/_security/enroll/node", "");
                    enrollResponse = client.execute(
                        "GET",
                        enrollNodeUrl,
                        new SecureString(enrollmentToken.getApiKey().toCharArray()),
                        () -> null,
                        CommandLineHttpClient::responseBuilder
                    );
                    break;
                } catch (Exception e) {
                    terminal.errorPrint(
                        Terminal.Verbosity.NORMAL,
                        "Unable to communicate with the node on " + enrollNodeUrl + ". Error was " + e.getMessage()
                    );
                }
            }
            if (enrollResponse == null || enrollResponse.getHttpStatus() != 200) {
                UserException userException = new UserException(
                    ExitCodes.UNAVAILABLE,
                    "Aborting enrolling to cluster. "
                        + "Could not communicate with the node on any of the addresses from the enrollment token. All of "
                        + enrollmentToken.getBoundAddress()
                        + " were attempted."
                );
                try {
                    deleteDirectory(tempGeneratedTlsCertsDir);
                } catch (Exception ex) {
                    userException.addSuppressed(ex);
                }
                throw userException;
            }
            final Map<String, Object> responseMap = enrollResponse.getResponseBody();
            if (responseMap == null) {
                UserException userException = new UserException(
                    ExitCodes.DATA_ERROR,
                    "Aborting enrolling to cluster. Empty response when calling the enroll node API (" + enrollNodeUrl + ")"
                );
                try {
                    deleteDirectory(tempGeneratedTlsCertsDir);
                } catch (Exception ex) {
                    userException.addSuppressed(ex);
                }
                throw userException;
            }
            final List<String> missingFields = new ArrayList<>();
            final String httpCaKeyPem = (String) responseMap.get("http_ca_key");
            if (Strings.isNullOrEmpty(httpCaKeyPem)) {
                missingFields.add("http_ca_key");
            }
            final String httpCaCertPem = (String) responseMap.get("http_ca_cert");
            if (Strings.isNullOrEmpty(httpCaCertPem)) {
                missingFields.add("http_ca_cert");
            }
            final String transportKeyPem = (String) responseMap.get("transport_key");
            if (Strings.isNullOrEmpty(transportKeyPem)) {
                missingFields.add("transport_key");
            }
            final String transportCaCertPem = (String) responseMap.get("transport_ca_cert");
            if (Strings.isNullOrEmpty(transportCaCertPem)) {
                missingFields.add("transport_ca_cert");
            }
            final String transportCertPem = (String) responseMap.get("transport_cert");
            if (Strings.isNullOrEmpty(transportCertPem)) {
                missingFields.add("transport_cert");
            }
            transportAddresses = getTransportAddresses(responseMap);
            if (null == transportAddresses || transportAddresses.isEmpty()) {
                missingFields.add("nodes_addresses");
            }
            if (false == missingFields.isEmpty()) {
                UserException userException = new UserException(
                    ExitCodes.DATA_ERROR,
                    "Aborting enrolling to cluster. Invalid response when calling the enroll node API ("
                        + enrollNodeUrl
                        + "). "
                        + "The following fields were empty or missing : "
                        + missingFields
                );
                try {
                    deleteDirectory(tempGeneratedTlsCertsDir);
                } catch (Exception ex) {
                    userException.addSuppressed(ex);
                }
                throw userException;
            }
            transportCaCert = parseCertificateFromPem(transportCaCertPem, terminal);
            httpCaKey = parseKeyFromPem(httpCaKeyPem, terminal);
            httpCaCert = parseCertificateFromPem(httpCaCertPem, terminal);
            transportKey = parseKeyFromPem(transportKeyPem, terminal);
            transportCert = parseCertificateFromPem(transportCertPem, terminal);
        } else {
            // this is the initial node, generate HTTP CA key/certificate and transport layer key/certificate ourselves
            try {
                transportAddresses = List.of();
                // self-signed CA for transport layer
                final KeyPair transportCaKeyPair = CertGenUtils.generateKeyPair(TRANSPORT_CA_KEY_SIZE);
                final PrivateKey transportCaKey = transportCaKeyPair.getPrivate();
                transportCaCert = CertGenUtils.generateSignedCertificate(
                    transportCaPrincipal,
                    null,
                    transportCaKeyPair,
                    null,
                    null,
                    true,
                    TRANSPORT_CA_CERTIFICATE_DAYS,
                    SIGNATURE_ALGORITHM,
                    null,
                    Set.of()
                );
                // transport key/certificate
                final KeyPair transportKeyPair = CertGenUtils.generateKeyPair(TRANSPORT_KEY_SIZE);
                transportKey = transportKeyPair.getPrivate();
                transportCert = CertGenUtils.generateSignedCertificate(
                    certificatePrincipal,
                    null, // transport only validates certificates not the hostname
                    transportKeyPair,
                    transportCaCert,
                    transportCaKey,
                    false,
                    TRANSPORT_CERTIFICATE_DAYS,
                    SIGNATURE_ALGORITHM,
                    null,
                    Set.of()
                );

                final KeyPair httpCaKeyPair = CertGenUtils.generateKeyPair(HTTP_CA_KEY_SIZE);
                httpCaKey = httpCaKeyPair.getPrivate();
                // self-signed CA
                httpCaCert = CertGenUtils.generateSignedCertificate(
                    httpCaPrincipal,
                    null,
                    httpCaKeyPair,
                    null,
                    null,
                    true,
                    HTTP_CA_CERTIFICATE_DAYS,
                    SIGNATURE_ALGORITHM,
                    buildKeyUsage(DEFAULT_CA_KEY_USAGE),
                    Set.of()
                );
            } catch (Throwable t) {
                try {
                    deleteDirectory(tempGeneratedTlsCertsDir);
                } catch (Exception ex) {
                    t.addSuppressed(ex);
                }
                // this is an error which mustn't be ignored during node startup
                // the exit code for unhandled Exceptions is "1"
                throw t;
            }
        }
        try {
            final KeyPair httpKeyPair = CertGenUtils.generateKeyPair(HTTP_KEY_SIZE);
            httpKey = httpKeyPair.getPrivate();
            // non-CA
            httpCert = CertGenUtils.generateSignedCertificate(
                certificatePrincipal,
                getSubjectAltNames(env.settings()),
                httpKeyPair,
                httpCaCert,
                httpCaKey,
                false,
                HTTP_CERTIFICATE_DAYS,
                SIGNATURE_ALGORITHM,
                buildKeyUsage(DEFAULT_CERT_KEY_USAGE),
                Set.of(new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth))
            );

            // the HTTP CA PEM file is provided "just in case". The node doesn't use it, but clients (configured manually, outside of the
            // enrollment process) might indeed need it, and it is currently impossible to retrieve it
            fullyWriteFile(
                tempGeneratedTlsCertsDir,
                HTTP_AUTOGENERATED_CA_NAME + ".crt",
                false,
                inReconfigureMode ? ELASTICSEARCH_GROUP_OWNER : null,
                stream -> {
                    try (
                        JcaPEMWriter pemWriter = new JcaPEMWriter(
                            new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8))
                        )
                    ) {
                        pemWriter.writeObject(httpCaCert);
                    }
                }
            );
        } catch (Throwable t) {
            try {
                deleteDirectory(tempGeneratedTlsCertsDir);
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            // this is an error which mustn't be ignored during node startup
            // the exit code for unhandled Exceptions is "1"
            throw t;
        }

        // save the existing keystore before replacing
        final Path keystoreBackupPath = env.configDir()
            .resolve(
                String.format(Locale.ROOT, KeyStoreWrapper.KEYSTORE_FILENAME + ".%d.orig", autoConfigDate.toInstant().getEpochSecond())
            );
        if (Files.exists(keystorePath)) {
            try {
                Files.copy(keystorePath, keystoreBackupPath, StandardCopyOption.COPY_ATTRIBUTES);
            } catch (Throwable t) {
                try {
                    deleteDirectory(tempGeneratedTlsCertsDir);
                } catch (Exception ex) {
                    t.addSuppressed(ex);
                }
                throw t;
            }
        }

        final SetOnce<SecureString> nodeKeystorePassword = new SetOnce<>();
        try (KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configDir(), () -> {
            nodeKeystorePassword.set(new SecureString(terminal.readSecret("")));
            return nodeKeystorePassword.get().clone();
        })) {
            // do not overwrite keystore entries
            // instead expect the admin to manually remove them herself
            if (nodeKeystore.getSettingNames().contains("xpack.security.transport.ssl.keystore.secure_password")
                || nodeKeystore.getSettingNames().contains("xpack.security.transport.ssl.truststore.secure_password")
                || nodeKeystore.getSettingNames().contains("xpack.security.http.ssl.keystore.secure_password")) {
                // this error condition is akin to condition of existing configuration in the yml file
                // this is not a fresh install and the admin has something planned for Security
                // Even though this is probably invalid configuration, do NOT fix it, let the node fail to start in its usual way.
                // Still display a message, because this can be tricky to figure out (why auto-conf did not run) if by mistake.
                throw new UserException(
                    ExitCodes.CONFIG,
                    "Aborting auto configuration because the node keystore contains password settings already"
                );
            }
            try (SecureString transportKeystorePassword = newKeystorePassword()) {
                KeyStore transportKeystore = KeyStore.getInstance("PKCS12");
                transportKeystore.load(null);
                // the PKCS12 keystore and the contained private key use the same password
                transportKeystore.setKeyEntry(
                    TRANSPORT_KEY_KEYSTORE_ENTRY,
                    transportKey,
                    transportKeystorePassword.getChars(),
                    new Certificate[] { transportCert }
                );
                transportKeystore.setCertificateEntry(TRANSPORT_CA_CERT_KEYSTORE_ENTRY, transportCaCert);
                fullyWriteFile(
                    tempGeneratedTlsCertsDir,
                    TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12",
                    false,
                    inReconfigureMode ? ELASTICSEARCH_GROUP_OWNER : null,
                    stream -> transportKeystore.store(stream, transportKeystorePassword.getChars())
                );
                nodeKeystore.setString("xpack.security.transport.ssl.keystore.secure_password", transportKeystorePassword.getChars());
                // we use the same PKCS12 file for the keystore and the truststore
                nodeKeystore.setString("xpack.security.transport.ssl.truststore.secure_password", transportKeystorePassword.getChars());
            }
            try (SecureString httpKeystorePassword = newKeystorePassword()) {
                KeyStore httpKeystore = KeyStore.getInstance("PKCS12");
                httpKeystore.load(null);
                // the keystore contains both the node's and the CA's private keys
                // both keys are encrypted using the same password as the PKCS12 keystore they're contained in
                httpKeystore.setKeyEntry(
                    HTTP_CA_KEY_KEYSTORE_ENTRY,
                    httpCaKey,
                    httpKeystorePassword.getChars(),
                    new Certificate[] { httpCaCert }
                );
                httpKeystore.setKeyEntry(
                    HTTP_KEY_KEYSTORE_ENTRY,
                    httpKey,
                    httpKeystorePassword.getChars(),
                    new Certificate[] { httpCert, httpCaCert }
                );
                fullyWriteFile(
                    tempGeneratedTlsCertsDir,
                    HTTP_AUTOGENERATED_KEYSTORE_NAME + ".p12",
                    false,
                    inReconfigureMode ? ELASTICSEARCH_GROUP_OWNER : null,
                    stream -> httpKeystore.store(stream, httpKeystorePassword.getChars())
                );
                nodeKeystore.setString("xpack.security.http.ssl.keystore.secure_password", httpKeystorePassword.getChars());
            }
            // finally overwrites the node keystore (if the keystores have been successfully written)
            nodeKeystore.save(env.configDir(), nodeKeystorePassword.get() == null ? new char[0] : nodeKeystorePassword.get().getChars());
        } catch (Throwable t) {
            // restore keystore to revert possible keystore bootstrap
            try {
                if (Files.exists(keystoreBackupPath)) {
                    Files.move(
                        keystoreBackupPath,
                        keystorePath,
                        StandardCopyOption.REPLACE_EXISTING,
                        StandardCopyOption.ATOMIC_MOVE,
                        StandardCopyOption.COPY_ATTRIBUTES
                    );
                } else {
                    Files.deleteIfExists(keystorePath);
                }
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            try {
                deleteDirectory(tempGeneratedTlsCertsDir);
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            throw t;
        } finally {
            if (nodeKeystorePassword.get() != null) {
                nodeKeystorePassword.get().close();
            }
        }

        try {
            // all certs and keys have been generated in the temp certs dir, therefore:
            // 1. backup (move) any previously existing tls certs dir (this backup is NOT removed when auto-conf finishes)
            if (Files.exists(env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME))) {
                moveDirectory(
                    env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME),
                    env.configDir()
                        .resolve(
                            String.format(
                                Locale.ROOT,
                                TLS_GENERATED_CERTS_DIR_NAME + ".%d.orig",
                                autoConfigDate.toInstant().getEpochSecond()
                            )
                        )
                );
            }
            // 2. move the newly populated temp certs dir to its permanent static dir name
            moveDirectory(tempGeneratedTlsCertsDir, env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME));
        } catch (Throwable t) {
            // restore keystore to revert possible keystore bootstrap
            try {
                if (Files.exists(keystoreBackupPath)) {
                    Files.move(
                        keystoreBackupPath,
                        keystorePath,
                        StandardCopyOption.REPLACE_EXISTING,
                        StandardCopyOption.ATOMIC_MOVE,
                        StandardCopyOption.COPY_ATTRIBUTES
                    );
                } else {
                    Files.deleteIfExists(keystorePath);
                }
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            // revert any previously existing TLS certs
            try {
                if (Files.exists(
                    env.configDir()
                        .resolve(
                            String.format(
                                Locale.ROOT,
                                TLS_GENERATED_CERTS_DIR_NAME + ".%d.orig",
                                autoConfigDate.toInstant().getEpochSecond()
                            )
                        )
                )) {
                    moveDirectory(
                        env.configDir()
                            .resolve(
                                String.format(
                                    Locale.ROOT,
                                    TLS_GENERATED_CERTS_DIR_NAME + ".%d.orig",
                                    autoConfigDate.toInstant().getEpochSecond()
                                )
                            ),
                        env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME)
                    );
                }
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            try {
                deleteDirectory(tempGeneratedTlsCertsDir);
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            throw t;
        }

        try {
            // final Environment to be used in the lambda below
            final Environment localFinalEnv = env;
            final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss", Locale.ROOT);
            List<String> existingConfigLines = Files.readAllLines(ymlPath, StandardCharsets.UTF_8);
            fullyWriteFile(env.configDir(), "elasticsearch.yml", true, stream -> {
                try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8))) {
                    // start with the existing config lines
                    for (String line : existingConfigLines) {
                        bw.write(line);
                        bw.newLine();
                    }
                    bw.newLine();
                    bw.write(AUTO_CONFIGURATION_START_MARKER);
                    bw.newLine();
                    bw.write("#");
                    bw.newLine();
                    bw.write("# The following settings, TLS certificates, and keys have been automatically      ");
                    bw.newLine();
                    bw.write("# generated to configure Elasticsearch security features on ");
                    bw.write(autoConfigDate.format(dateTimeFormatter));
                    bw.newLine();
                    bw.write("#");
                    // TODO add link to docs
                    bw.newLine();
                    bw.write("# --------------------------------------------------------------------------------");
                    bw.newLine();
                    bw.newLine();
                    bw.write("# Enable security features");
                    bw.newLine();
                    bw.write(XPackSettings.SECURITY_ENABLED.getKey() + ": true");
                    bw.newLine();
                    bw.newLine();
                    // Set enrollment mode to true unless user explicitly set it to false themselves
                    if (false == (localFinalEnv.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())
                        && false == XPackSettings.ENROLLMENT_ENABLED.get(localFinalEnv.settings()))) {
                        bw.write(XPackSettings.ENROLLMENT_ENABLED.getKey() + ": true");
                        bw.newLine();
                        bw.newLine();
                    }
                    bw.write("# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents");
                    bw.newLine();
                    bw.write("xpack.security.http.ssl:");
                    bw.newLine();
                    bw.write("  enabled: true");
                    bw.newLine();
                    bw.write("  keystore.path: " + TLS_GENERATED_CERTS_DIR_NAME + "/" + HTTP_AUTOGENERATED_KEYSTORE_NAME + ".p12");
                    bw.newLine();
                    bw.newLine();
                    bw.write("# Enable encryption and mutual authentication between cluster nodes");
                    bw.newLine();
                    bw.write("xpack.security.transport.ssl:");
                    bw.newLine();
                    bw.write("  enabled: true");
                    bw.newLine();
                    bw.write("  verification_mode: certificate");
                    bw.newLine();
                    bw.write("  keystore.path: " + TLS_GENERATED_CERTS_DIR_NAME + "/" + TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12");
                    bw.newLine();
                    // we use the keystore as a truststore in order to minimize the number of auto-generated resources,
                    // and also because a single file is more idiomatic to the scheme of a shared secret between the cluster nodes
                    // no one should only need the TLS cert without the associated key for the transport layer
                    bw.write("  truststore.path: " + TLS_GENERATED_CERTS_DIR_NAME + "/" + TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12");
                    if (inEnrollmentMode) {
                        bw.newLine();
                        bw.write("# Discover existing nodes in the cluster");
                        bw.newLine();
                        bw.write(
                            DISCOVERY_SEED_HOSTS_SETTING.getKey()
                                + ": ["
                                + transportAddresses.stream().map(p -> '"' + p + '"').collect(Collectors.joining(", "))
                                + "]"
                        );
                        bw.newLine();
                    } else {
                        // we have configured TLS on the transport layer with newly generated certs and keys,
                        // hence this node cannot form a multi-node cluster
                        // if we don't set the following the node might trip the discovery bootstrap check
                        if (false == DiscoveryModule.isSingleNodeDiscovery(localFinalEnv.settings())
                            && false == ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.exists(localFinalEnv.settings())) {
                            bw.newLine();
                            bw.write("# Create a new cluster with the current node only");
                            bw.newLine();
                            bw.write("# Additional nodes can still join the cluster later");
                            bw.newLine();
                            bw.write(
                                ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey()
                                    + ": "
                                    + initialMasterNodesSettingValue(localFinalEnv)
                            );
                            bw.newLine();
                        }
                    }

                    // if any address settings have been set, assume the admin has thought it through wrt to addresses,
                    // and don't try to be smart and mess with that
                    if (false == (localFinalEnv.settings().hasValue(HttpTransportSettings.SETTING_HTTP_HOST.getKey())
                        || localFinalEnv.settings().hasValue(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey())
                        || localFinalEnv.settings().hasValue(HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST.getKey())
                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey())
                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey())
                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_PUBLISH_HOST_SETTING.getKey()))) {
                        bw.newLine();
                        bw.write("# Allow HTTP API connections from anywhere");
                        bw.newLine();
                        bw.write("# Connections are encrypted and require user authentication");
                        bw.newLine();
                        bw.write(HttpTransportSettings.SETTING_HTTP_HOST.getKey() + ": 0.0.0.0");
                        bw.newLine();
                    }
                    if (false == (localFinalEnv.settings().hasValue(TransportSettings.HOST.getKey())
                        || localFinalEnv.settings().hasValue(TransportSettings.BIND_HOST.getKey())
                        || localFinalEnv.settings().hasValue(TransportSettings.PUBLISH_HOST.getKey())
                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey())
                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey())
                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_PUBLISH_HOST_SETTING.getKey()))) {
                        bw.newLine();
                        bw.write("# Allow other nodes to join the cluster from anywhere");
                        bw.newLine();
                        bw.write("# Connections are encrypted and mutually authenticated");
                        bw.newLine();
                        if (false == inEnrollmentMode
                            || false == anyRemoteHostNodeAddress(transportAddresses, NetworkUtils.getAllAddresses())) {
                            bw.write("#");
                        }
                        bw.write(TransportSettings.HOST.getKey() + ": 0.0.0.0");
                        bw.newLine();
                    }
                    bw.newLine();
                    bw.write(AUTO_CONFIGURATION_END_MARKER);
                    bw.newLine();
                }
            });
        } catch (Throwable t) {
            try {
                if (Files.exists(keystoreBackupPath)) {
                    Files.move(keystoreBackupPath, keystorePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
                } else {
                    // this removes a statically named file, so it is potentially dangerous
                    Files.deleteIfExists(keystorePath);
                }
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            try {
                // this removes a statically named directory, so it is potentially dangerous
                deleteDirectory(env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME));
            } catch (Exception ex) {
                t.addSuppressed(ex);
            }
            Path backupCertsDir = env.configDir()
                .resolve(
                    String.format(Locale.ROOT, TLS_GENERATED_CERTS_DIR_NAME + ".%d.orig", autoConfigDate.toInstant().getEpochSecond())
                );
            if (Files.exists(backupCertsDir)) {
                moveDirectory(backupCertsDir, env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME));
            }
            throw t;
        }
        // only delete the backed-up keystore file if all went well, because the new keystore contains its entries
        if (Files.exists(keystoreBackupPath)) {
            Files.delete(keystoreBackupPath);
        }
    }