5-infrastructure/modules/base-env/boa-gke.tf (200 lines of code) (raw):
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
gke_settings = {
gke1 = {
name = "gke-1-boa-${local.envs[var.env].short}-${var.location_primary}"
subnetwork = var.gke_cluster_1_subnet_name
ip_range_pods = var.gke_cluster_1_range_name_pods
ip_range_services = var.gke_cluster_1_range_name_services
master_ipv4_cidr_block = var.gke_cluster_1_cidr_block
default_max_pods_per_node = var.max_pods_per_node
region = var.location_primary
node_pool_min_count = 2
node_pool_max_count = 3
machine_type = "e2-standard-4"
master_authorized_networks = [
{
cidr_block = element([for subnet_ip_range in flatten([for subnet in data.google_compute_subnetwork.subnet : subnet.secondary_ip_range if length(subnet.secondary_ip_range) > 0 && subnet.name == var.gke_cluster_2_subnet_name]) : subnet_ip_range.ip_cidr_range if subnet_ip_range.range_name == var.gke_cluster_2_range_name_pods], 0)
display_name = "cluster 2 pods to cluster 1 controlplane"
}
]
},
gke2 = {
name = "gke-2-boa-${local.envs[var.env].short}-${var.location_secondary}"
subnetwork = var.gke_cluster_2_subnet_name
ip_range_pods = var.gke_cluster_2_range_name_pods
ip_range_services = var.gke_cluster_2_range_name_services
master_ipv4_cidr_block = var.gke_cluster_2_cidr_block
default_max_pods_per_node = var.max_pods_per_node
region = var.location_secondary
machine_type = "e2-standard-4"
node_pool_min_count = 2
node_pool_max_count = 3
master_authorized_networks = [
{
cidr_block = element([for subnet_ip_range in flatten([for subnet in data.google_compute_subnetwork.subnet : subnet.secondary_ip_range if length(subnet.secondary_ip_range) > 0 && subnet.name == var.gke_cluster_1_subnet_name]) : subnet_ip_range.ip_cidr_range if subnet_ip_range.range_name == var.gke_cluster_1_range_name_pods], 0)
display_name = "cluster 1 pods to cluster 2 controlplane"
}
]
},
mci = {
name = "mci-boa-${local.envs[var.env].short}-${var.location_primary}"
subnetwork = var.gke_mci_cluster_subnet_name
ip_range_pods = var.gke_mci_cluster_range_name_pods
ip_range_services = var.gke_mci_cluster_range_name_services
master_ipv4_cidr_block = var.gke_mci_cluster_cidr_block
default_max_pods_per_node = var.max_pods_per_node
region = var.location_primary
machine_type = "e2-standard-2"
node_pool_min_count = 1
node_pool_max_count = 3
master_authorized_networks = []
}
}
bin_auth_attestors = [for attestor in var.bin_auth_attestor_names : "projects/${var.bin_auth_attestor_project_id}/attestors/${attestor}"]
allowlist_patterns = ["quay.io/random-containers/*", "k8s.gcr.io/more-random/*", "gcr.io/${var.boa_gke_project_id}/*", "gcr.io/config-management-release/*"] # Example
}
module "sink_gke" {
source = "terraform-google-modules/log-export/google"
version = "~> 6.0"
destination_uri = module.log_destination.destination_uri
filter = "resource.type:(k8s_cluster OR k8s_container OR gce_target_https_proxy OR gce_url_map OR http_load_balancer OR gce_target_https_proxy OR gce_backend_service OR gce_instance OR gce_forwarding_rule OR gce_health_check OR service_account OR global OR audited_resource OR project)"
log_sink_name = "sink-boa-${local.envs[var.env].short}-gke-to-ops"
parent_resource_id = var.boa_gke_project_id
parent_resource_type = "project"
unique_writer_identity = true
}
module "bastion" {
source = "../bastion"
project_id = var.boa_gke_project_id
bastion_name = "gce-bastion-${lower(var.bastion_zone)}-01"
bastion_zone = var.bastion_zone
bastion_service_account_name = "boa-gce-bastion-${local.envs[var.env].short}-sa"
bastion_members = var.bastion_members
vpc_name = var.shared_vpc_name
bastion_subnet = var.bastion_subnet_name
bastion_region = var.location_secondary
network_project_id = var.gcp_shared_vpc_project_id
repo_project_id = var.bin_auth_attestor_project_id
}
data "google_project" "gke_project" {
project_id = var.boa_gke_project_id
}
module "clusters" {
source = "terraform-google-modules/kubernetes-engine/google//modules/safer-cluster"
version = "~> 14.0.1"
for_each = local.gke_settings
project_id = var.boa_gke_project_id
network_project_id = var.gcp_shared_vpc_project_id
network = var.shared_vpc_name
name = each.value.name
subnetwork = each.value.subnetwork
ip_range_pods = each.value.ip_range_pods
ip_range_services = each.value.ip_range_services
master_ipv4_cidr_block = each.value.master_ipv4_cidr_block
default_max_pods_per_node = each.value.default_max_pods_per_node
region = each.value.region
master_authorized_networks = concat(each.value.master_authorized_networks,
[
{
cidr_block = module.bastion.cidr_range,
display_name = "bastion subnet to cluster controlplane"
}
]
)
cluster_resource_labels = {
"mesh_id" = "proj-${data.google_project.gke_project.number}"
}
node_pools_tags = {
"np-${each.value.region}" : ["boa-${each.key}-cluster", "allow-google-apis", "egress-internet", "boa-cluster", "allow-lb"]
}
node_pools = [
{
name = "np-${each.value.region}",
auto_repair = true,
auto_upgrade = true,
enable_secure_boot = true,
image_type = "COS_CONTAINERD",
machine_type = each.value.machine_type,
max_count = each.value.node_pool_max_count,
min_count = each.value.node_pool_min_count,
node_metadata = "GKE_METADATA_SERVER"
}
]
node_pools_oauth_scopes = {
"all" : [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/devstorage.read_only"
],
"default-node-pool" : []
}
compute_engine_service_account = "boa-gke-nodes-${local.envs[var.env].short}-gsa@${var.boa_gke_project_id}.iam.gserviceaccount.com"
}
resource "google_binary_authorization_policy" "policy" {
project = var.boa_gke_project_id
global_policy_evaluation_mode = "ENABLE"
default_admission_rule {
evaluation_mode = "REQUIRE_ATTESTATION"
enforcement_mode = var.enforce_bin_auth_policy ? "ENFORCED_BLOCK_AND_AUDIT_LOG" : "DRYRUN_AUDIT_LOG_ONLY"
require_attestations_by = local.bin_auth_attestors
}
dynamic "admission_whitelist_patterns" {
for_each = local.allowlist_patterns
content {
name_pattern = admission_whitelist_patterns.value
}
}
}
/******************************************
Cloud Armor policy
*****************************************/
resource "google_compute_security_policy" "cloud-armor-xss-policy" {
name = "cloud-armor-xss-policy"
project = var.boa_gke_project_id
rule {
action = "deny(403)"
priority = "1000"
match {
expr {
expression = "evaluatePreconfiguredExpr('xss-stable')"
}
}
description = "Cloud Armor policy to prevent cross-site scripting attacks."
}
rule {
action = "allow"
priority = "2147483647"
match {
versioned_expr = "SRC_IPS_V1"
config {
src_ip_ranges = ["*"]
}
}
description = "default rule"
}
}
/******************************************
External IP Address
*****************************************/
resource "google_compute_global_address" "external_ip_for_http_load_balancing" {
name = "mci-ip"
project = var.boa_gke_project_id
address_type = "EXTERNAL"
description = "External IP address for HTTP load balancing on MCI subnet."
}