diff --git a/README.md b/README.md index e95bc6a..26659c9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,6 @@ EC2 instance sizing can be configured in [terraform/environments/staging/main.tf ## AWS Setting the Kubernetes context ``` aws eks update-kubeconfig --name --region us-west-2 -aws eks update-kubeconfig --name --region us-west-2 --role-arn ``` ## Workin with Kubernetes diff --git a/kubernetes/terraform/environments/development/main.tf b/kubernetes/terraform/environments/development/main.tf index c58eac1..d6ddc9a 100644 --- a/kubernetes/terraform/environments/development/main.tf +++ b/kubernetes/terraform/environments/development/main.tf @@ -16,26 +16,8 @@ module "kubernetes" { region = "<% index .Params `region` %>" # Authenticate with the EKS cluster via the cluster id - cluster_name = "<% .Name %>-dev-cluster" - - # Assume-role policy used by monitoring fluentd daemonset - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json + cluster_name = "<% .Name %>-dev-<% index .Params `region` %>" external_dns_zone = "<% index .Params `stagingHost` %>" external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID - external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-staging-workers" ] -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } } diff --git a/kubernetes/terraform/environments/production/main.tf b/kubernetes/terraform/environments/production/main.tf index 1be2599..5b5544a 100644 --- a/kubernetes/terraform/environments/production/main.tf +++ b/kubernetes/terraform/environments/production/main.tf @@ -16,26 +16,8 @@ module "kubernetes" { region = "<% index .Params `region` %>" # Authenticate with the EKS cluster via the cluster id - cluster_name = "<% .Name %>-production-cluster" - - # Assume-role policy used by monitoring fluentd daemonset - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json + cluster_name = "<% .Name %>-production-<% index .Params `region` %>" external_dns_zone = "<% index .Params `productionHost` %>" external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID - external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-production-workers" ] -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } } diff --git a/kubernetes/terraform/environments/staging/main.tf b/kubernetes/terraform/environments/staging/main.tf index f328fa7..dd0547c 100644 --- a/kubernetes/terraform/environments/staging/main.tf +++ b/kubernetes/terraform/environments/staging/main.tf @@ -20,26 +20,8 @@ module "kubernetes" { region = "<% index .Params `region` %>" # Authenticate with the EKS cluster via the cluster id - cluster_name = "<% .Name %>-staging-cluster" - - # Assume-role policy used by monitoring fluentd daemonset - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json + cluster_name = "<% .Name %>-staging-<% index .Params `region` %>" external_dns_zone = "<% index .Params `stagingHost` %>" external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID - external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-<% .Name %>-staging-<% index .Params `region` %>-workers" ] -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } } diff --git a/kubernetes/terraform/modules/kubernetes/README.md b/kubernetes/terraform/modules/kubernetes/README.md index 5ef98d7..6dafc9e 100644 --- a/kubernetes/terraform/modules/kubernetes/README.md +++ b/kubernetes/terraform/modules/kubernetes/README.md @@ -4,24 +4,30 @@ This Terraform module contains configuration to provision kubernetes resources. +## Core Components + +[Nginx Ingress Controller](https://github.com/kubernetes/ingress-nginx/) +Use Nginx as a reverse proxy and load balancer for your cluster. This will create an AWS load balancer (ELB/ALB/NLB) and whenever an ingress is created to route traffic to your application, the controller will make sure the LB is up to date and sending traffic where it needs to go. + +[External DNS](https://github.com/kubernetes-sigs/external-dns) +For any ingresses that are added to route traffic for hosts, external-dns will automatically create DNS records for those hosts and point it to the LB created by the ingress controller. +This makes is extremely easy to bring up a new site at a specific domain or subdomain. + +[Cert Manager](https://github.com/jetstack/cert-manager) +For any ingresses that specify that they need TLS, cert-manager will automatically provision a certificate using Lets Encrypt, and handle renewing it automatically on a regular basis. +Alongside external-dns, this allows you to make sure your new domains are always secured using HTTPS. + +[Cloudwatch Agent/Fluentd](https://github.com/fluent/fluentd) +A unified logging layer, Fluentd handles capturing all log output from your cluster and routing it to various sources like Cloudwatch, Elasticsearch, etc. + + +... + ## Organization ``` main.tf - Configuration entrypoint. - ingress/ - Confioguration required to provision nginx-ingress-controller. - main.tf - provider.tf - variables.tf - monitoring/ - Configuration required to provision cluster monitoring. - main.tf - provider.tf - variables.tf - fluentd/ - main.tf - files/ - ... - cloudwatch/ - main.tf - files/ - ... -``` \ No newline at end of file + external_dns.tf - Set up external-dns + ingress/ - Provision nginx-ingress-controller. + monitoring/ - Provision cluster monitoring (cloudwatch agent and fluentd). +``` diff --git a/kubernetes/terraform/modules/kubernetes/external_dns.tf b/kubernetes/terraform/modules/kubernetes/external_dns.tf index a31405f..8bfd683 100644 --- a/kubernetes/terraform/modules/kubernetes/external_dns.tf +++ b/kubernetes/terraform/modules/kubernetes/external_dns.tf @@ -1,28 +1,18 @@ -# Trust relationship -data "aws_iam_policy_document" "external_dns_trust_relationship" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } - - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = var.external_dns_assume_roles - } - } +# Create a role using oidc to map service accounts +module "iam_assumable_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> v2.6.0" + create_role = true + role_name = "<% .Name %>-k8s-${var.environment}-external-dns" + provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") + role_policy_arns = [aws_iam_policy.external_dns.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:external-dns"] } -# external-dns role -resource "aws_iam_role" "external_dns_role" { - name = "k8s-external-dns-role" - assume_role_policy = data.aws_iam_policy_document.external_dns_trust_relationship.json +resource "aws_iam_policy" "external_dns" { + name_prefix = "external-dns" + description = "EKS external-dns policy for cluster ${var.cluster_name}" + policy = data.aws_iam_policy_document.external_dns_policy_doc.json } data "aws_iam_policy_document" "external_dns_policy_doc" { @@ -48,16 +38,13 @@ data "aws_iam_policy_document" "external_dns_policy_doc" { } } -resource "aws_iam_role_policy" "external_dns_policy" { - name = "k8s-external-dns-policy" - role = aws_iam_role.external_dns_role.id - policy = data.aws_iam_policy_document.external_dns_policy_doc.json -} - resource "kubernetes_service_account" "external_dns" { metadata { - name = "external-dns" - namespace = "kube-system" + name = "external-dns" + namespace = "kube-system" + annotations = { + "eks.amazonaws.com/role-arn" = module.iam_assumable_role.this_iam_role_arn + } } } @@ -116,7 +103,7 @@ resource "kubernetes_deployment" "external_dns" { "app" = "external-dns", } annotations = { - "iam.amazonaws.com/role" = "k8s-external-dns-role", + "eks.amazonaws.com/role-arn" = module.iam_assumable_role.this_iam_role_arn } } spec { @@ -124,17 +111,21 @@ resource "kubernetes_deployment" "external_dns" { name = "external-dns" image = "registry.opensource.zalan.do/teapot/external-dns:latest" args = [ - "--source=service", "--source=ingress", "--domain-filter=${var.external_dns_zone}", # Give access only to the specified zone "--provider=aws", "--aws-zone-type=public", "--policy=upsert-only", # Prevent ExternalDNS from deleting any records "--registry=txt", - "--txt-owner-id=${var.external_dns_owner_id}", # ID of txt record to manage state + "--txt-owner-id=${var.cluster_name}", # ID of txt record to manage state + "--aws-batch-change-size=2", # Set the batch size to 2 so that a single record failure won't block other updates ] } + security_context { + fs_group = 65534 + } + service_account_name = "external-dns" automount_service_account_token = true } diff --git a/kubernetes/terraform/modules/kubernetes/kube2iam/README.md b/kubernetes/terraform/modules/kubernetes/kube2iam/README.md deleted file mode 100644 index dcfe37f..0000000 --- a/kubernetes/terraform/modules/kubernetes/kube2iam/README.md +++ /dev/null @@ -1,6 +0,0 @@ -kube2iam -= - -Provide IAM credentials to containers running inside a kubernetes cluster based on annotations. - -Will be deprecated soon in favour of: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/. \ No newline at end of file diff --git a/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf b/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf deleted file mode 100644 index 75d16d8..0000000 --- a/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf +++ /dev/null @@ -1,105 +0,0 @@ -resource "kubernetes_service_account" "kube2iam" { - metadata { - name = "kube2iam" - namespace = "kube-system" - } -} - -resource "kubernetes_cluster_role" "kube2iam" { - metadata { - name = "kube2iam" - } - rule { - verbs = ["get", "watch", "list"] - api_groups = [""] - resources = ["namespaces", "pods"] - } -} - -resource "kubernetes_cluster_role_binding" "kube2iam" { - metadata { - name = "kube2iam" - } - subject { - kind = "ServiceAccount" - name = "kube2iam" - namespace = "kube-system" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "kube2iam" - } -} - -resource "kubernetes_daemonset" "kube2iam" { - metadata { - name = "kube2iam" - namespace = "kube-system" - labels = { - app = "kube2iam" - } - } - spec { - selector { - match_labels = { - name = "kube2iam" - } - } - template { - metadata { - labels = { - name = "kube2iam" - } - } - spec { - container { - name = "kube2iam" - image = "jtblin/kube2iam:0.10.8" - args = [ - "--auto-discover-base-arn", - "--auto-discover-default-role", - "--iptables=true", - "--host-ip=$(HOST_IP)", - "--host-interface=eni+", - # "--node=$(NODE_NAME)", - "--use-regional-sts-endpoint", - "--log-level=info" - ] - port { - name = "http" - host_port = 8181 - container_port = 8181 - } - env { - name = "HOST_IP" - value_from { - field_ref { - field_path = "status.podIP" - } - } - } - # env { - # name = "NODE_NAME" - # value_from { - # field_ref { - # field_path = "spec.nodeName" - # } - # } - # } - env { - name = "AWS_REGION" - value = var.region - } - security_context { - privileged = true - } - } - service_account_name = "kube2iam" - automount_service_account_token = true - host_network = true - } - } - } -} - diff --git a/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf b/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf deleted file mode 100644 index dc049cf..0000000 --- a/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf +++ /dev/null @@ -1,7 +0,0 @@ -variable "region" { - description = "AWS Region" -} - -variable "environment" { - description = "Environment" -} \ No newline at end of file diff --git a/kubernetes/terraform/modules/kubernetes/main.tf b/kubernetes/terraform/modules/kubernetes/main.tf index cc01642..0cf69ed 100644 --- a/kubernetes/terraform/modules/kubernetes/main.tf +++ b/kubernetes/terraform/modules/kubernetes/main.tf @@ -1,14 +1,8 @@ -module "kube2iam" { - source = "./kube2iam" - environment = var.environment - region = var.region -} module "monitoring" { source = "./monitoring" environment = var.environment region = var.region - assume_role_policy = var.assume_role_policy cluster_name = var.cluster_name } diff --git a/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf b/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf index a58c791..92e537b 100644 --- a/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf +++ b/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf @@ -1,7 +1,10 @@ resource "kubernetes_service_account" "cloudwatch_agent" { metadata { - name = "cloudwatch-agent" - namespace = "amazon-cloudwatch" + name = "cloudwatch-agent" + namespace = "amazon-cloudwatch" + annotations = { + "eks.amazonaws.com/role-arn" = module.iam_assumable_role_cloudwatch.this_iam_role_arn + } } depends_on = [kubernetes_namespace.amazon_cloudwatch] } @@ -93,7 +96,7 @@ resource "kubernetes_daemonset" "cloudwatch_agent" { metadata { labels = { name = "cloudwatch-agent" } annotations = { - "iam.amazonaws.com/role" = "k8s-${var.environment}-monitoring" + "eks.amazonaws.com/role-arn" = module.iam_assumable_role_cloudwatch.this_iam_role_arn } } spec { @@ -209,6 +212,11 @@ resource "kubernetes_daemonset" "cloudwatch_agent" { mount_path = "/dev/disk" } } + + security_context { + fs_group = 65534 + } + termination_grace_period_seconds = 60 service_account_name = "cloudwatch-agent" automount_service_account_token = true diff --git a/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf b/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf index 24ceca0..d4ce37c 100644 --- a/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf +++ b/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf @@ -28,8 +28,11 @@ resource "kubernetes_config_map" "cluster_info" { resource "kubernetes_service_account" "fluentd" { metadata { - name = "fluentd" - namespace = "amazon-cloudwatch" + name = "fluentd" + namespace = "amazon-cloudwatch" + annotations = { + "eks.amazonaws.com/role-arn" = module.iam_assumable_role_fluentd.this_iam_role_arn + } } depends_on = [kubernetes_namespace.amazon_cloudwatch] } @@ -101,8 +104,7 @@ resource "kubernetes_daemonset" "fluentd_cloudwatch" { k8s-app = "fluentd-cloudwatch" } annotations = { - configHash = "8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825", - "iam.amazonaws.com/role" = "k8s-${var.environment}-monitoring" + "eks.amazonaws.com/role-arn" = module.iam_assumable_role_fluentd.this_iam_role_arn } } spec { @@ -215,6 +217,11 @@ resource "kubernetes_daemonset" "fluentd_cloudwatch" { mount_path = "/var/log/dmesg" } } + + security_context { + fs_group = 65534 + } + termination_grace_period_seconds = 30 service_account_name = "fluentd" automount_service_account_token = true diff --git a/kubernetes/terraform/modules/kubernetes/monitoring/main.tf b/kubernetes/terraform/modules/kubernetes/monitoring/main.tf index 840913c..4fba905 100644 --- a/kubernetes/terraform/modules/kubernetes/monitoring/main.tf +++ b/kubernetes/terraform/modules/kubernetes/monitoring/main.tf @@ -1,7 +1,32 @@ -resource "aws_iam_role" "k8s_monitoring" { - name = "<% .Name %>-k8s-${var.environment}-monitoring" - assume_role_policy = var.assume_role_policy - force_detach_policies = true +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} + +# Create a role using oidc to map service accounts +module "iam_assumable_role_cloudwatch" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> v2.6.0" + create_role = true + role_name = "<% .Name %>-k8s-${var.environment}-cloudwatch" + provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") + role_policy_arns = [data.aws_iam_policy.CloudWatchAgentServerPolicy.arn] + oidc_fully_qualified_subjects = [ "system:serviceaccount:amazon-cloudwatch:cloudwatch-agent" ] +} + +# Create a role using oidc to map service accounts +module "iam_assumable_role_fluentd" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> v2.6.0" + create_role = true + role_name = "<% .Name %>-k8s-${var.environment}-fluentd" + provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") + role_policy_arns = [data.aws_iam_policy.CloudWatchAgentServerPolicy.arn] + oidc_fully_qualified_subjects = [ "system:serviceaccount:amazon-cloudwatch:fluentd" ] +} + + +data "aws_iam_policy" "CloudWatchAgentServerPolicy" { + arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" } # Create amazon-cloudwatch kubernetes namespace for fluentd/cloudwatchagent @@ -13,12 +38,3 @@ resource "kubernetes_namespace" "amazon_cloudwatch" { } } } - -data "aws_iam_policy" "CloudWatchAgentServerPolicy" { - arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" -} - -resource "aws_iam_role_policy_attachment" "k8s_monitoring_role_policy" { - role = aws_iam_role.k8s_monitoring.id - policy_arn = data.aws_iam_policy.CloudWatchAgentServerPolicy.arn -} diff --git a/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf b/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf index 93e911d..95a3ab3 100644 --- a/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf +++ b/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf @@ -6,10 +6,6 @@ variable "environment" { description = "Environment" } -variable "assume_role_policy" { - description = "Assume-role policy" -} - variable "cluster_name" { description = "Cluster name" -} \ No newline at end of file +} diff --git a/kubernetes/terraform/modules/kubernetes/provider.tf b/kubernetes/terraform/modules/kubernetes/provider.tf index c4afddd..a70cf1a 100644 --- a/kubernetes/terraform/modules/kubernetes/provider.tf +++ b/kubernetes/terraform/modules/kubernetes/provider.tf @@ -1,4 +1,4 @@ -# https://github.com/terraform-providers/terraform-provider-kubernetes/issues/161#issuecomment-461190931 +data "aws_caller_identity" "current" {} data "aws_eks_cluster" "cluster" { name = var.cluster_name diff --git a/kubernetes/terraform/modules/kubernetes/variables.tf b/kubernetes/terraform/modules/kubernetes/variables.tf index f75de7f..973dc82 100644 --- a/kubernetes/terraform/modules/kubernetes/variables.tf +++ b/kubernetes/terraform/modules/kubernetes/variables.tf @@ -10,19 +10,10 @@ variable "cluster_name" { description = "Kubernetes cluster name" } -variable "assume_role_policy" { - description = "Assume-role policy for monitoring" -} - variable "external_dns_zone" { - description = "R53 zone that external-dns will have access to" + description = "Domain of R53 zone that external-dns will have access to" } variable "external_dns_owner_id" { description = "Unique id of the TXT record that external-dns will use to store state (can just be a uuid)" } - -variable "external_dns_assume_roles" { - description = "List of roles that should be able to assume the external dns role (most likely the role of the cluster worker nodes)" - type = list(string) -} diff --git a/terraform/environments/production/main.tf b/terraform/environments/production/main.tf index 4fc01a9..2abb380 100644 --- a/terraform/environments/production/main.tf +++ b/terraform/environments/production/main.tf @@ -22,13 +22,14 @@ module "production" { ecr_repositories = ["production"] # EKS configuration - eks_worker_instance_type = "t2.small" - eks_worker_asg_min_size = 3 - eks_worker_asg_max_size = 6 + eks_cluster_version = "1.15" + eks_worker_instance_type = "t3.medium" + eks_worker_asg_min_size = 2 + eks_worker_asg_max_size = 4 # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html - # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 - eks_worker_ami = "ami-07be7092831897fd6" + # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.15%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 + eks_worker_ami = "ami-0e710550577202c55" # Hosting configuration s3_hosting_buckets = [ diff --git a/terraform/environments/staging/main.tf b/terraform/environments/staging/main.tf index aed4167..b694ca4 100644 --- a/terraform/environments/staging/main.tf +++ b/terraform/environments/staging/main.tf @@ -22,13 +22,14 @@ module "staging" { ecr_repositories = [ "gql-server" ] # EKS configuration - eks_worker_instance_type = "t2.small" - eks_worker_asg_min_size = 2 - eks_worker_asg_max_size = 6 + eks_cluster_version = "1.15" + eks_worker_instance_type = "t3.medium" + eks_worker_asg_min_size = 1 + eks_worker_asg_max_size = 3 # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html - # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 - eks_worker_ami = "ami-07be7092831897fd6" + # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.15%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 + eks_worker_ami = "ami-0e710550577202c55" # Hosting configuration s3_hosting_buckets = [ diff --git a/terraform/modules/eks/main.tf b/terraform/modules/eks/main.tf index e3d7dd1..70899ba 100644 --- a/terraform/modules/eks/main.tf +++ b/terraform/modules/eks/main.tf @@ -1,18 +1,35 @@ +# Set up the terraform provider +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.11" +} + # Create KubernetesAdmin role for aws-iam-authenticator resource "aws_iam_role" "kubernetes_admin_role" { - name = "<% .Name %>-kubernetes-admin" + name = "<% .Name %>-kubernetes-admin-${var.environment}" assume_role_policy = var.assume_role_policy description = "Kubernetes administrator role (for AWS IAM Authenticator)" } module "eks" { source = "terraform-aws-modules/eks/aws" - version = "6.0.2" + version = "10.0.0" cluster_name = var.cluster_name - cluster_version = "1.14" + cluster_version = var.cluster_version subnets = var.private_subnets vpc_id = var.vpc_id + enable_irsa = true worker_groups = [ { @@ -31,7 +48,7 @@ module "eks" { map_roles = [ { - rolearn = "arn:aws:iam::${var.iam_account_id}:role/<% .Name %>-kubernetes-admin" + rolearn = "arn:aws:iam::${var.iam_account_id}:role/<% .Name %>-kubernetes-admin-${var.environment}" username = "<% .Name %>-kubernetes-admin" groups = ["system:masters"] }, @@ -39,11 +56,10 @@ module "eks" { cluster_iam_role_name = "k8s-${var.cluster_name}-cluster" workers_role_name = "k8s-${var.cluster_name}-workers" - # TODO, determine if this should be true/false - manage_aws_auth = true + # Unfortunately fluentd doesn't yet support oidc auth so we need to grant it to the worker nodes + workers_additional_policies = ["arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"] write_kubeconfig = false - write_aws_auth_config = false tags = { environment = var.environment diff --git a/terraform/modules/eks/variables.tf b/terraform/modules/eks/variables.tf index 537e2fc..40a32c1 100644 --- a/terraform/modules/eks/variables.tf +++ b/terraform/modules/eks/variables.tf @@ -10,8 +10,12 @@ variable "cluster_name" { description = "Name to be given to the EKS cluster" } +variable "cluster_version" { + description = "EKS cluster version number to use. Incrementing this will start a cluster upgrade" +} + variable "assume_role_policy" { - description = "IAM policy document for AssumeRole" + description = "IAM policy document for AssumeRole. Controls access to the kubernetes admin serviceaccount" } variable "private_subnets" { diff --git a/terraform/modules/environment/main.tf b/terraform/modules/environment/main.tf index ff4cb57..7655d4b 100644 --- a/terraform/modules/environment/main.tf +++ b/terraform/modules/environment/main.tf @@ -16,6 +16,7 @@ module "vpc" { # Data sources for EKS IAM data "aws_caller_identity" "current" {} +# Use this role to limit access to the k8s admin serviceaccount data "aws_iam_policy_document" "assumerole_root_policy" { statement { actions = ["sts:AssumeRole"] @@ -35,6 +36,8 @@ module "eks" { project = var.project environment = var.environment cluster_name = local.kubernetes_cluster_name + cluster_version = var.eks_cluster_version + iam_account_id = data.aws_caller_identity.current.account_id assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json @@ -47,15 +50,6 @@ module "eks" { worker_ami = var.eks_worker_ami # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html } -module "kube2iam" { - source = "../../modules/kube2iam" - - environment = var.environment - eks_worker_iam_role_arn = module.eks.worker_iam_role_arn - eks_worker_iam_role_name = module.eks.worker_iam_role_name - iam_account_id = data.aws_caller_identity.current.account_id -} - data "aws_iam_user" "ci_user" { user_name = "ci-user" # Should have been created in the bootstrap process } diff --git a/terraform/modules/environment/variables.tf b/terraform/modules/environment/variables.tf index ced3ab1..f6dc570 100644 --- a/terraform/modules/environment/variables.tf +++ b/terraform/modules/environment/variables.tf @@ -20,6 +20,10 @@ variable "ecr_repositories" { type = list(string) } +variable "eks_cluster_version" { + description = "EKS cluster version number to use. Incrementing this will start a cluster upgrade" +} + variable "eks_worker_instance_type" { description = "Instance type for the EKS workers" } diff --git a/terraform/modules/kube2iam/README.md b/terraform/modules/kube2iam/README.md deleted file mode 100644 index dcfe37f..0000000 --- a/terraform/modules/kube2iam/README.md +++ /dev/null @@ -1,6 +0,0 @@ -kube2iam -= - -Provide IAM credentials to containers running inside a kubernetes cluster based on annotations. - -Will be deprecated soon in favour of: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/. \ No newline at end of file diff --git a/terraform/modules/kube2iam/main.tf b/terraform/modules/kube2iam/main.tf deleted file mode 100644 index 2c433b7..0000000 --- a/terraform/modules/kube2iam/main.tf +++ /dev/null @@ -1,130 +0,0 @@ -# Allow the worker nodes to assume a role we are creating below -data "aws_iam_policy_document" "k8s_worker_assumerole_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = [var.eks_worker_iam_role_arn] - } - } -} - -# Policy to allow worker nodes to assume roles starting with "k8s-" -data "aws_iam_policy_document" "node_assume_kube2iam_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - resources = ["arn:aws:iam::${var.iam_account_id}:role/k8s-*"] - } -} - -# Add the above policy to the worker role -resource "aws_iam_role_policy" "node_kube2iam_policy" { - name = "eks-node-kube2iam-policy" - role = var.eks_worker_iam_role_name - policy = data.aws_iam_policy_document.node_assume_kube2iam_role.json -} - -# This is now done with the kubernetes terraform provider, see the kubernetes/kube2iam module. -# # Execute the kubernetes manifest required to create the kube2iam daemonset -# resource "null_resource" "kube2iam" { -# provisioner "local-exec" { -# command = "kubectl apply -f ${path.root}/kubernetes/kube2iam.yaml --kubeconfig ${path.root}/output/kubeconfig_${var.environment}" -# } -# # TODO: Module-aware dependencies not yet supported - https://github.com/hashicorp/terraform/issues/17101 -# # depends_on = ["module.eks"] -# } - -### Kube2IAM roles to map to pods ### -# These can be referenced in an annotation in a kubernetes deployment manifest file - -## ALB Ingress Controller -# Create a role and establish a trust relationship with the worker nodes -resource "aws_iam_role" "k8s_worker_alb_ingress_controller_role" { - name = "<% .Name %>-k8s-alb-ingress-controller" - assume_role_policy = data.aws_iam_policy_document.k8s_worker_assumerole_policy.json - force_detach_policies = true -} - -# Policy allowing access to specific AWS resources -data "aws_iam_policy_document" "k8s_alb_ingress_controller_access_policy" { - statement { - actions = [ - "acm:DescribeCertificate", - "acm:ListCertificates", - "acm:GetCertificate", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:DeleteTags", - "ec2:DeleteSecurityGroup", - "ec2:DescribeAccountAttributes", - "ec2:DescribeInstances", - "ec2:DescribeInstanceStatus", - "ec2:DescribeInternetGateways", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeTags", - "ec2:DescribeVpcs", - "ec2:ModifyInstanceAttribute", - "ec2:ModifyNetworkInterfaceAttribute", - "ec2:RevokeSecurityGroupIngress", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateRule", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:DeleteListener", - "elasticloadbalancing:DeleteLoadBalancer", - "elasticloadbalancing:DeleteRule", - "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:DeregisterTargets", - "elasticloadbalancing:DescribeListenerCertificates", - "elasticloadbalancing:DescribeListeners", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeLoadBalancerAttributes", - "elasticloadbalancing:DescribeRules", - "elasticloadbalancing:DescribeSSLPolicies", - "elasticloadbalancing:DescribeTags", - "elasticloadbalancing:DescribeTargetGroups", - "elasticloadbalancing:DescribeTargetGroupAttributes", - "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:ModifyListener", - "elasticloadbalancing:ModifyLoadBalancerAttributes", - "elasticloadbalancing:ModifyRule", - "elasticloadbalancing:ModifyTargetGroup", - "elasticloadbalancing:ModifyTargetGroupAttributes", - "elasticloadbalancing:RegisterTargets", - "elasticloadbalancing:RemoveTags", - "elasticloadbalancing:SetIpAddressType", - "elasticloadbalancing:SetSecurityGroups", - "elasticloadbalancing:SetSubnets", - "elasticloadbalancing:SetWebACL", - "iam:GetServerCertificate", - "iam:ListServerCertificates", - "waf-regional:GetWebACLForResource", - "waf-regional:GetWebACL", - "waf-regional:AssociateWebACL", - "waf-regional:DisassociateWebACL", - "waf:GetWebACL", - "tag:GetResources", - "tag:TagResources", - "cognito-idp:DescribeUserPoolClient", - ] - - resources = ["*"] - } - - statement { - actions = ["iam:CreateServiceLinkedRole"] - resources = ["arn:aws:iam::${var.iam_account_id}:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing"] - } -} - -# Add the above policy to the created role -resource "aws_iam_role_policy" "k8s_worker_alb_ingress_controller_role_policy" { - name = "worker-alb-ingress-controller-policy" - role = aws_iam_role.k8s_worker_alb_ingress_controller_role.id - policy = data.aws_iam_policy_document.k8s_alb_ingress_controller_access_policy.json -} diff --git a/terraform/modules/kube2iam/variables.tf b/terraform/modules/kube2iam/variables.tf deleted file mode 100644 index 5f210b9..0000000 --- a/terraform/modules/kube2iam/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "eks_worker_iam_role_arn" { - description = "The ARN of the EKS worker IAM role" -} - -variable "eks_worker_iam_role_name" { - description = "The name of the EKS worker IAM role" -} - -variable "iam_account_id" { - description = "Account ID of the current IAM user" -} - -variable "environment" { - description = "The environment (dev/staging/prod)" -} \ No newline at end of file diff --git a/terraform/modules/kube2iam/versions.tf b/terraform/modules/kube2iam/versions.tf deleted file mode 100644 index ac97c6a..0000000 --- a/terraform/modules/kube2iam/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/terraform/modules/vpc/main.tf b/terraform/modules/vpc/main.tf index 97baa56..838ecc7 100644 --- a/terraform/modules/vpc/main.tf +++ b/terraform/modules/vpc/main.tf @@ -4,10 +4,10 @@ module "vpc" { name = "${var.project}-${var.environment}-vpc" cidr = "10.10.0.0/16" - azs = ["${var.region}a", "${var.region}b", "${var.region}c"] # Most regions have 3+ azs - private_subnets = ["10.10.32.0/19", "10.10.64.0/19", "10.10.96.0/19"] - public_subnets = ["10.10.1.0/24", "10.10.2.0/24", "10.10.3.0/24"] - database_subnets = ["10.10.10.0/24", "10.10.11.0/24", "10.10.12.0/24"] + azs = ["${var.region}a", "${var.region}b"] # Most regions have 3+ azs + private_subnets = ["10.10.32.0/19", "10.10.64.0/19"] + public_subnets = ["10.10.1.0/24", "10.10.2.0/24"] + database_subnets = ["10.10.10.0/24", "10.10.11.0/24"] # Allow kubernetes ALB ingress controller to auto-detect private_subnet_tags = {