From d10c919c859de7f4c03b3fc7febebf4140eb3f4f Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 11 Aug 2020 18:17:41 -0400 Subject: [PATCH 1/2] terraform: upgrading from 0.12 to 0.13 --- templates/README.md | 2 -- templates/kubernetes/terraform/modules/kubernetes/README.md | 2 +- .../kubernetes/terraform/modules/kubernetes/cert_manager.tf | 2 +- .../terraform/modules/kubernetes/cluster_autoscaler.tf | 2 +- .../kubernetes/terraform/modules/kubernetes/external_dns.tf | 2 +- .../terraform/modules/kubernetes/monitoring/main.tf | 4 ++-- templates/terraform/README.md | 4 ++-- templates/terraform/bootstrap/secrets/main.tf | 2 +- templates/terraform/environments/prod/main.tf | 2 +- templates/terraform/environments/stage/main.tf | 2 +- templates/terraform/modules/certificate/versions.tf | 2 +- templates/terraform/modules/database/main.tf | 2 +- templates/terraform/modules/database/versions.tf | 2 +- templates/terraform/modules/ecr/versions.tf | 2 +- templates/terraform/modules/eks/versions.tf | 2 +- templates/terraform/modules/environment/versions.tf | 2 +- templates/terraform/modules/s3_hosting/versions.tf | 2 +- templates/terraform/modules/secret/versions.tf | 2 +- templates/terraform/modules/vpc/versions.tf | 2 +- zero-module.yml | 1 - 20 files changed, 20 insertions(+), 23 deletions(-) diff --git a/templates/README.md b/templates/README.md index a11a53c..e98635f 100644 --- a/templates/README.md +++ b/templates/README.md @@ -1,5 +1,3 @@ - - # Overview Your infrastructure should be up and running, your terraform repository is the source of truth for your infrastructure, here is [a list of components and resources][zero-resource-list] that comes with the EKS-stack diff --git a/templates/kubernetes/terraform/modules/kubernetes/README.md b/templates/kubernetes/terraform/modules/kubernetes/README.md index 2eadbbe..47bf47d 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/README.md +++ b/templates/kubernetes/terraform/modules/kubernetes/README.md @@ -38,7 +38,7 @@ The `cert_manager.tf` config has a good example of using this in practice. To al ``` module "iam_assumable_role_my_role_name" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "~> v2.14.0" create_role = true role_name = "my-role-name" provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") diff --git a/templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf b/templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf index 72e27fa..cbca5b6 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf @@ -87,7 +87,7 @@ resource "helm_release" "cert_manager" { # Create a role using oidc to map service accounts module "iam_assumable_role_cert_manager" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "~> v2.14.0" create_role = true role_name = "<% .Name %>-k8s-${var.environment}-cert-manager" provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") diff --git a/templates/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf b/templates/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf index 25b631b..48a45e1 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf @@ -36,7 +36,7 @@ resource "helm_release" "cluster_autoscaler" { # Create a role using oidc to map service accounts module "iam_assumable_role_cluster_autoscaler" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "~> v2.14.0" create_role = true role_name = "<% .Name %>-k8s-${var.environment}-cluster-autoscaler" provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") diff --git a/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf b/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf index 1295194..99f9e30 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf @@ -1,7 +1,7 @@ # Create a role using oidc to map service accounts module "iam_assumable_role_external_dns" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "~> v2.14.0" create_role = true role_name = "<% .Name %>-k8s-${var.environment}-external-dns" provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf index 4fba905..6ebb151 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf @@ -5,7 +5,7 @@ data "aws_eks_cluster" "cluster" { # Create a role using oidc to map service accounts module "iam_assumable_role_cloudwatch" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "~> v2.14.0" create_role = true role_name = "<% .Name %>-k8s-${var.environment}-cloudwatch" provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") @@ -16,7 +16,7 @@ module "iam_assumable_role_cloudwatch" { # Create a role using oidc to map service accounts module "iam_assumable_role_fluentd" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "~> v2.6.0" + version = "~> v2.14.0" create_role = true role_name = "<% .Name %>-k8s-${var.environment}-fluentd" provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "") diff --git a/templates/terraform/README.md b/templates/terraform/README.md index 28f02fb..ca978ba 100644 --- a/templates/terraform/README.md +++ b/templates/terraform/README.md @@ -1,12 +1,12 @@ ## Guidelines & Style Convention Summary - All Terraform configuration should be formatted with `terraform fmt` before being accepted into this repository. -- This repository is Terraform version >= 0.12, as such, leverage features from this release whenever possible. +- This repository is Terraform version >= 0.13, as such, leverage features from this release whenever possible. See https://www.terraform.io/upgrade-guides/0-12.html for more information. - Leverage community-maintained Terraform modules whenever possible. - Attempt to minimize duplication whenever possible, but only within reason -- sometimes duplication is an acceptable solution. - Follow style conventions described in `docs/guide.pdf` whenever possible. -- Whenever possible, inject resources down versus referencing resources across modules. This has been made easier with new features in v0.12. +- Whenever possible, inject resources down versus referencing resources across modules. This has been made easier with new features in v0.13. - Whenever possible, define the types of variables. ### Module Conventions diff --git a/templates/terraform/bootstrap/secrets/main.tf b/templates/terraform/bootstrap/secrets/main.tf index 55c8b08..2c8d838 100644 --- a/templates/terraform/bootstrap/secrets/main.tf +++ b/templates/terraform/bootstrap/secrets/main.tf @@ -5,7 +5,7 @@ provider "aws" { terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } locals { diff --git a/templates/terraform/environments/prod/main.tf b/templates/terraform/environments/prod/main.tf index cb3d945..2d2fb6d 100644 --- a/templates/terraform/environments/prod/main.tf +++ b/templates/terraform/environments/prod/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" backend "s3" { bucket = "<% .Name %>-prod-terraform-state" key = "infrastructure/terraform/environments/production/main" diff --git a/templates/terraform/environments/stage/main.tf b/templates/terraform/environments/stage/main.tf index 7d47ece..7671de2 100644 --- a/templates/terraform/environments/stage/main.tf +++ b/templates/terraform/environments/stage/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" backend "s3" { bucket = "<% .Name %>-stage-terraform-state" key = "infrastructure/terraform/environments/staging/main" diff --git a/templates/terraform/modules/certificate/versions.tf b/templates/terraform/modules/certificate/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/certificate/versions.tf +++ b/templates/terraform/modules/certificate/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/database/main.tf b/templates/terraform/modules/database/main.tf index 8418050..075e6d5 100644 --- a/templates/terraform/modules/database/main.tf +++ b/templates/terraform/modules/database/main.tf @@ -41,7 +41,7 @@ data "aws_secretsmanager_secret_version" "rds_master_secret" { module "rds" { source = "terraform-aws-modules/rds/aws" - version = "2.14.0" + version = "2.17.0" identifier = "${var.project}-${var.environment}" diff --git a/templates/terraform/modules/database/versions.tf b/templates/terraform/modules/database/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/database/versions.tf +++ b/templates/terraform/modules/database/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/ecr/versions.tf b/templates/terraform/modules/ecr/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/ecr/versions.tf +++ b/templates/terraform/modules/ecr/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/eks/versions.tf b/templates/terraform/modules/eks/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/eks/versions.tf +++ b/templates/terraform/modules/eks/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/environment/versions.tf b/templates/terraform/modules/environment/versions.tf index d9b6f79..6b6318d 100644 --- a/templates/terraform/modules/environment/versions.tf +++ b/templates/terraform/modules/environment/versions.tf @@ -1,3 +1,3 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/s3_hosting/versions.tf b/templates/terraform/modules/s3_hosting/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/s3_hosting/versions.tf +++ b/templates/terraform/modules/s3_hosting/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/secret/versions.tf b/templates/terraform/modules/secret/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/secret/versions.tf +++ b/templates/terraform/modules/secret/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/templates/terraform/modules/vpc/versions.tf b/templates/terraform/modules/vpc/versions.tf index ac97c6a..2606a5a 100644 --- a/templates/terraform/modules/vpc/versions.tf +++ b/templates/terraform/modules/vpc/versions.tf @@ -1,4 +1,4 @@ terraform { - required_version = ">= 0.12" + required_version = ">= 0.13" } diff --git a/zero-module.yml b/zero-module.yml index 2bb7f33..550c9c7 100644 --- a/zero-module.yml +++ b/zero-module.yml @@ -70,4 +70,3 @@ parameters: - field: randomSeed label: Random seed that will be shared between projects to come up with deterministic resource names execute: uuidgen | head -c 8 - From 3246ead4c4853a9e556852720695e089f0904f4a Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 11 Aug 2020 18:57:44 -0400 Subject: [PATCH 2/2] database: adding RDS options of mysql --- .../modules/kubernetes/database-service.tf | 5 +- templates/terraform/environments/prod/main.tf | 1 + .../terraform/environments/stage/main.tf | 1 + templates/terraform/modules/database/main.tf | 73 ++++++++++++++++++- .../terraform/modules/database/variables.tf | 4 + .../terraform/modules/environment/main.tf | 1 + .../modules/environment/variables.tf | 5 ++ zero-module.yml | 5 ++ 8 files changed, 90 insertions(+), 5 deletions(-) diff --git a/templates/kubernetes/terraform/modules/kubernetes/database-service.tf b/templates/kubernetes/terraform/modules/kubernetes/database-service.tf index 8236986..9d4dd9e 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/database-service.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/database-service.tf @@ -9,9 +9,12 @@ resource "kubernetes_namespace" "app_namespace" { } resource "kubernetes_service" "app_db" { + ## this should match the deployable backend's name/namespace + ## it uses this service to connect and create application user + ## https://github.com/commitdev/zero-deployable-backend/blob/b2cee21982b1e6a0ac9996e2a1bf214e5bf10ab5/db-ops/create-db-user.sh#L6 metadata { namespace = kubernetes_namespace.app_namespace.metadata[0].name - name = "<% .Name %>" + name = "database" } spec { type = "ExternalName" diff --git a/templates/terraform/environments/prod/main.tf b/templates/terraform/environments/prod/main.tf index 2d2fb6d..b7e9ce6 100644 --- a/templates/terraform/environments/prod/main.tf +++ b/templates/terraform/environments/prod/main.tf @@ -39,6 +39,7 @@ module "prod" { domain_name = "<% index .Params `productionHostRoot` %>" # DB configuration + database = "<% index .Params `database` %>" db_instance_class = "db.t3.small" db_storage_gb = 100 diff --git a/templates/terraform/environments/stage/main.tf b/templates/terraform/environments/stage/main.tf index 7671de2..2f0681d 100644 --- a/templates/terraform/environments/stage/main.tf +++ b/templates/terraform/environments/stage/main.tf @@ -40,6 +40,7 @@ module "stage" { vpc_use_single_nat_gateway = true # DB configuration + database = "<% index .Params `database` %>" db_instance_class = "db.t3.small" db_storage_gb = 20 } diff --git a/templates/terraform/modules/database/main.tf b/templates/terraform/modules/database/main.tf index 075e6d5..43718ed 100644 --- a/templates/terraform/modules/database/main.tf +++ b/templates/terraform/modules/database/main.tf @@ -9,13 +9,19 @@ module "rds_security_group" { number_of_computed_ingress_with_source_security_group_id = 1 computed_ingress_with_source_security_group_id = [ - { + var.database == "postgres" ? { from_port = 5432 to_port = 5432 protocol = "tcp" description = "PostgreSQL from EKS" source_security_group_id = "${var.allowed_security_group_id}" - }, + }:{ + from_port = 3306 + to_port = 3306 + protocol = "tcp" + description = "MYSQL from EKS" + source_security_group_id = "${var.allowed_security_group_id}" + } ] egress_rules = ["all-all"] @@ -39,7 +45,8 @@ data "aws_secretsmanager_secret_version" "rds_master_secret" { secret_id = data.aws_secretsmanager_secret.rds_master_secret.name } -module "rds" { +module "rds_postgres" { + count = var.database_engine == "postgres" ? 1 : 0 source = "terraform-aws-modules/rds/aws" version = "2.17.0" @@ -78,11 +85,69 @@ module "rds" { # Enhanced monitoring performance_insights_enabled = true create_monitoring_role = true - monitoring_role_name = "${var.project}-${var.environment}-rds-monitoring-role" + monitoring_role_name = "${var.project}-${var.environment}-rds-postgres-monitoring-role" + monitoring_interval = "30" + + tags = { + Name = "${var.project}-${var.environment}-rds-postgres" + Env = "${var.environment}" + } + depends_on = [module.rds_security_group] +} + +module "rds_mysql" { + count = var.database_engine == "mysql" ? 1 : 0 + source = "terraform-aws-modules/rds/aws" + version = "2.17.0" + + identifier = "${var.project}-${var.environment}" + + engine = "mysql" + engine_version = "5.7" + instance_class = var.instance_class + allocated_storage = var.storage_gb + storage_encrypted = true + + name = "${replace(var.project, "-", "")}" + username = "master_user" + password = "${data.aws_secretsmanager_secret_version.rds_master_secret.secret_string}" + port = "3306" + + vpc_security_group_ids = ["${module.rds_security_group.this_security_group_id}"] + + maintenance_window = "Mon:00:00-Mon:03:00" + backup_window = "03:00-06:00" + + # disable backups to create DB faster in non-production environments + backup_retention_period = var.environment == "production" ? 30 : 0 + + # Subnet is created by the vpc module + create_db_subnet_group = false + db_subnet_group_name = "${var.project}-${var.environment}-vpc" + + # DB parameter and option group + family = "mysql5.7" + major_engine_version = "5.7" + + final_snapshot_identifier = "final-snapshot" + deletion_protection = true + + # Enhanced monitoring + # Seems like mysql doesnt have performance insight on this instance size + # Amazon RDS for MySQL + # 8.0.17 and higher 8.0 versions, version 5.7.22 and higher 5.7 versions, + # and version 5.6.41 and higher 5.6 versions. Not supported for version 5.5. + # Not supported on the following DB instance classes: + # db.t2.micro, db.t2.small, db.t3.micro, db.t3.small, + # all db.m6g instance classes, and all db.r6g instance classes. + performance_insights_enabled = false + create_monitoring_role = true + monitoring_role_name = "${var.project}-${var.environment}-rds-mysql-monitoring-role" monitoring_interval = "30" tags = { Name = "${var.project}-${var.environment}-rds-postgres" Env = "${var.environment}" } + depends_on = [module.rds_security_group] } diff --git a/templates/terraform/modules/database/variables.tf b/templates/terraform/modules/database/variables.tf index eba2c3a..ebe7861 100644 --- a/templates/terraform/modules/database/variables.tf +++ b/templates/terraform/modules/database/variables.tf @@ -21,3 +21,7 @@ variable "instance_class" { variable "storage_gb" { description = "The amount of storage to allocate for the db, in GB" } + +variable "database_engine" { + description = "Which database engine to use, currently supports `postgres` or `mysql`" +} diff --git a/templates/terraform/modules/environment/main.tf b/templates/terraform/modules/environment/main.tf index b3883a2..bcc7a72 100644 --- a/templates/terraform/modules/environment/main.tf +++ b/templates/terraform/modules/environment/main.tf @@ -79,6 +79,7 @@ module "db" { allowed_security_group_id = module.eks.worker_security_group_id instance_class = var.db_instance_class storage_gb = var.db_storage_gb + database_engine = var.database } module "ecr" { diff --git a/templates/terraform/modules/environment/variables.tf b/templates/terraform/modules/environment/variables.tf index 08a1baa..e1e5b89 100644 --- a/templates/terraform/modules/environment/variables.tf +++ b/templates/terraform/modules/environment/variables.tf @@ -63,3 +63,8 @@ variable "vpc_use_single_nat_gateway" { type = bool default = true } + +variable "database" { + default = "postgres" + description = "Which database engine to use, currently supports postgres or mysql" +} diff --git a/zero-module.yml b/zero-module.yml index 550c9c7..95d11ee 100644 --- a/zero-module.yml +++ b/zero-module.yml @@ -61,6 +61,11 @@ parameters: type: regex value: '^([a-z0-9]+(-[a-z0-9]+)*\.)$' errorMessage: Invalid subdomain (cannot contain special chars & must end with a '.') + - field: database + label: Database engine to use (postgres) + options: + - "postgres" + - "mysql" - field: accountId label: AWS Account ID execute: aws sts get-caller-identity --query "Account" | tr -d '"'