Split base from modules

- Add infra as a example
- Introduces a dependency issue between adot operator and workload collector

examples/existing-cluster-with-base-and-infra/main.tf

@@ -48,7 +48,7 @@ locals {
 
 # deploys the base module
 module "eks_observability_accelerator" {
-  # source = "aws-ia/terrarom-aws-observability-accelerator?ref=dev"
+  # source = "aws-ia/terrarom-aws-observability-accelerator"
   source = "../../"
 
   aws_region     = var.aws_region
@@ -60,21 +60,17 @@ module "eks_observability_accelerator" {
   # reusing existing certificate manager? defaults to true
   enable_cert_manager = true
 
-  # # -- or enable opentelemetry operator
-  enable_opentelemetry_operator = false
-  #open_telemetry_operator_config = map() // custom config
-
   # creates a new AMP workspace, defaults to true
   enable_managed_prometheus = false
 
   # reusing existing AMP -- needs data source for alerting rules
-  managed_prometheus_id     = var.managed_prometheus_workspace_id
+  managed_prometheus_workspace_id     = var.managed_prometheus_workspace_id
-  managed_prometheus_region = null # defaults to the current region, useful for cross region scenarios (same account)
+  managed_prometheus_workspace_region = null # defaults to the current region, useful for cross region scenarios (same account)
 
   # sets up the AMP alert manager at the workspace level
   enable_alertmanager = true
 
-  # create a new Grafana workspace - TODO review design
+  # reusing existing Amazon Managed Grafana workspace
   enable_managed_grafana       = false
   managed_grafana_workspace_id = var.managed_grafana_workspace_id
   grafana_api_key              = var.grafana_api_key
@@ -84,9 +80,27 @@ module "eks_observability_accelerator" {
 
 module "workloads_infra" {
   source = "../../workloads/infra"
-  # source = "aws-ia/terrarom-aws-observability-accelerator/workloads/infra?ref=dev"
+  # source = "aws-ia/terrarom-aws-observability-accelerator/workloads/infra"
+
+  eks_cluster_id = module.eks_observability_accelerator.eks_cluster_id
+
+  dashboards_folder_id            = module.eks_observability_accelerator.grafana_dashboards_folder_id
+  managed_prometheus_workspace_id = module.eks_observability_accelerator.managed_prometheus_workspace_id
+
+  # TODO remove when Kevin's PR is live
+  managed_prometheus_workspace_endpoint = module.eks_observability_accelerator.managed_prometheus_workspace_endpoint
+  managed_prometheus_workspace_region   = module.eks_observability_accelerator.managed_prometheus_workspace_region
 
 
-  enable_infra_metrics = true
+  managed_grafana_workspace_endpoint = module.eks_observability_accelerator.managed_grafana_workspace_endpoint
-  # infra_metrics_config = {}
+
+  # TODO: manage with Secrets manager
+  grafana_api_key = var.grafana_api_key
+
+  # module custom configuration, check module documentation
+  # config               = {}
+
+  # depends_on = [
+  #   module.eks_observability_accelerator
+  # ]
 }

examples/existing-cluster-with-base-and-infra/outputs.tf

@@ -14,9 +14,9 @@ output "eks_cluster_version" {
 
 
 output "prometheus_endpoint" {
-  value = module.eks_observability_accelerator.prometheus_endpoint
+  value = module.eks_observability_accelerator.managed_prometheus_workspace_endpoint
 }
 
 output "prometheus_id" {
-  value = module.eks_observability_accelerator.prometheus_id
+  value = module.eks_observability_accelerator.managed_prometheus_workspace_id
 }

locals.tf

@@ -13,23 +13,6 @@ data "aws_grafana_workspace" "this" {
   workspace_id = var.managed_grafana_workspace_id
 }
 
-# resource "null_resource" "amg_api_key" {
-
-#   # Bootstrap script can run on any instance of the cluster
-#   # So we just choose the first in this case
-#   connection {
-#     host = element(aws_instance.cluster.*.public_ip, 0)
-#   }
-
-#   provisioner "remote-exec" {
-#     # requires aws-cli
-#     inline = [
-#       #"bootstrap-cluster.sh ${join(" ", aws_instance.cluster.*.private_ip)}",
-#       "aws grafana create-key",
-#     ]
-#   }
-# }
-
 
 locals {
   eks_oidc_issuer_url  = replace(data.aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer, "https://", "")
@@ -37,8 +20,8 @@ locals {
   eks_cluster_version  = data.aws_eks_cluster.eks_cluster.version
 
   # if region is not passed, we assume the current one
-  amp_ws_region   = coalesce(var.managed_prometheus_region, data.aws_region.current.name)
+  amp_ws_region   = coalesce(var.managed_prometheus_workspace_region, data.aws_region.current.name)
-  amp_ws_id       = var.enable_managed_prometheus ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_id
+  amp_ws_id       = var.enable_managed_prometheus ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_workspace_id
   amp_ws_endpoint = "https://aps-workspaces.${local.amp_ws_region}.amazonaws.com/workspaces/${local.amp_ws_id}/"
 
   # if region is not passed, we assume the current one
@@ -46,7 +29,8 @@ locals {
 
   # if grafana_workspace_id is supplied, we infer the endpoint from
   # computed region, else we create a new workspace
-  amg_ws_endpoint = var.managed_grafana_workspace_id == "" ? "https://${module.managed_grafana[0].workspace_endpoint}" : "https://${var.managed_grafana_workspace_id}.grafana-workspace.${local.amg_ws_region}.amazonaws.com"
+  amg_ws_endpoint = var.managed_grafana_workspace_id == "" ? "https://${module.managed_grafana[0].workspace_endpoint}" : "https://${data.aws_grafana_workspace.this[0].endpoint}"
+  amg_ws_id       = var.managed_grafana_workspace_id == "" ? module.managed_grafana[0].workspace_ : data.aws_grafana_workspace.this[0].endpoint
 
   # TODO when tf resource for AMG api keys are supported
   # create a short-lived api key on the fly if api_key is not provided

main.tf

@@ -1,13 +1,9 @@
 module "operator" {
-  source = "./modules/opentelemetry-operator"
+  source = "./modules/adot-operator"
 
   enable_cert_manager = var.enable_cert_manager
-
+  kubernetes_version  = local.eks_cluster_version
-  enable_amazon_eks_adot        = var.enable_amazon_eks_adot
+  addon_context       = local.context
-  enable_opentelemetry_operator = var.enable_opentelemetry_operator
-
-  kubernetes_version = local.eks_cluster_version
-  addon_context      = local.context
 }
 
 resource "aws_prometheus_workspace" "this" {

modules/adot-operator/locals.tf

@@ -0,0 +1,6 @@
+locals {
+  name                       = "adot"
+  eks_addon_role_name        = "eks:addon-manager"
+  eks_addon_clusterrole_name = "eks:addon-manager-otel"
+  addon_namespace            = "opentelemetry-operator-system"
+}

modules/adot-operator/main.tf

@@ -7,11 +7,10 @@ module "cert_manager" {
 }
 
 resource "kubernetes_namespace_v1" "adot" {
-  count = local.create_namespace ? 1 : 0
 
   metadata {
     # If using EKS addon, namespace must be "opentelemetry-operator-system"
-    name = var.enable_amazon_eks_adot ? local.addon_namespace : try(var.helm_config.namespace, local.addon_namespace)
+    name = local.addon_namespace
 
     labels = {
       # Prerequisite for EKS addon
@@ -21,19 +20,15 @@ resource "kubernetes_namespace_v1" "adot" {
 }
 
 data "aws_eks_addon_version" "this" {
-  count = var.enable_amazon_eks_adot ? 1 : 0
-
   addon_name         = local.name
   kubernetes_version = try(var.addon_config.kubernetes_version, var.kubernetes_version)
   most_recent        = try(var.addon_config.most_recent, true)
 }
 
 resource "aws_eks_addon" "adot" {
-  count = var.enable_amazon_eks_adot ? 1 : 0
-
   cluster_name             = var.addon_context.eks_cluster_id
   addon_name               = local.name
-  addon_version            = try(var.addon_config.addon_version, data.aws_eks_addon_version.this[0].version)
+  addon_version            = try(var.addon_config.addon_version, data.aws_eks_addon_version.this.version)
   resolve_conflicts        = try(var.addon_config.resolve_conflicts, "OVERWRITE")
   service_account_role_arn = try(var.addon_config.service_account_role_arn, null)
   preserve                 = try(var.addon_config.preserve, true)
@@ -43,8 +38,8 @@ resource "aws_eks_addon" "adot" {
     try(var.addon_config.tags, {}),
     # implicit dependency with roles
     {
-      RoleVersion        = try(kubernetes_role_v1.adot[0].metadata[0].resource_version, ""),
+      RoleVersion        = try(kubernetes_role_v1.adot.metadata[0].resource_version, ""),
-      ClusterRoleVersion = try(kubernetes_cluster_role_v1.adot[0].metadata[0].resource_version, "")
+      ClusterRoleVersion = try(kubernetes_cluster_role_v1.adot.metadata[0].resource_version, "")
     }
   )
 
@@ -52,11 +47,10 @@ resource "aws_eks_addon" "adot" {
 }
 
 resource "kubernetes_role_v1" "adot" {
-  count = var.enable_amazon_eks_adot ? 1 : 0
 
   metadata {
     name      = local.eks_addon_role_name
-    namespace = kubernetes_namespace_v1.adot[0].metadata[0].name
+    namespace = kubernetes_namespace_v1.adot.metadata[0].name
   }
 
   rule {
@@ -118,11 +112,10 @@ resource "kubernetes_role_v1" "adot" {
 }
 
 resource "kubernetes_role_binding_v1" "adot" {
-  count = var.enable_amazon_eks_adot ? 1 : 0
 
   metadata {
     name      = local.eks_addon_role_name
-    namespace = kubernetes_namespace_v1.adot[0].metadata[0].name
+    namespace = kubernetes_namespace_v1.adot.metadata[0].name
   }
 
   subject {
@@ -138,7 +131,6 @@ resource "kubernetes_role_binding_v1" "adot" {
 }
 
 resource "kubernetes_cluster_role_v1" "adot" {
-  count = var.enable_amazon_eks_adot ? 1 : 0
 
   metadata {
     name = local.eks_addon_clusterrole_name
@@ -153,7 +145,7 @@ resource "kubernetes_cluster_role_v1" "adot" {
   rule {
     api_groups     = [""]
     resources      = ["namespaces"]
-    resource_names = [kubernetes_namespace_v1.adot[0].metadata[0].name]
+    resource_names = [kubernetes_namespace_v1.adot.metadata[0].name]
     verbs          = ["create", "delete", "get", "list", "patch", "update", "watch"]
   }
   rule {
@@ -266,7 +258,6 @@ resource "kubernetes_cluster_role_v1" "adot" {
 }
 
 resource "kubernetes_cluster_role_binding_v1" "adot" {
-  count = var.enable_amazon_eks_adot ? 1 : 0
 
   metadata {
     name = local.eks_addon_clusterrole_name
@@ -283,15 +274,3 @@ resource "kubernetes_cluster_role_binding_v1" "adot" {
   }
 
 }
-
-
-module "helm_addon" {
-  source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
-  count  = var.enable_opentelemetry_operator ? 1 : 0
-
-  helm_config   = local.helm_config
-  irsa_config   = null
-  addon_context = var.addon_context
-
-  depends_on = [module.cert_manager]
-}

modules/adot-operator/variables.tf

@@ -34,15 +34,3 @@ variable "addon_config" {
   type        = any
   default     = {}
 }
-
-variable "enable_amazon_eks_adot" {
-  description = "Enable Amazon EKS ADOT add-on"
-  type        = bool
-  default     = true
-}
-
-variable "enable_opentelemetry_operator" {
-  description = "Enable opentelemetry operator addon"
-  type        = bool
-  default     = false
-}

modules/opentelemetry-operator/locals.tf

@@ -1,25 +0,0 @@
-locals {
-  name                       = "adot"
-  eks_addon_role_name        = "eks:addon-manager"
-  eks_addon_clusterrole_name = "eks:addon-manager-otel"
-  addon_namespace            = "opentelemetry-operator-system"
-
-  create_namespace = var.enable_opentelemetry_operator ? true : try(var.helm_config.create_namespace, true)
-  namespace        = local.create_namespace ? kubernetes_namespace_v1.adot[0].metadata[0].name : try(var.helm_config.namespace, local.addon_namespace)
-
-  default_helm_config = {
-    name        = "opentelemetry"
-    repository  = "https://open-telemetry.github.io/opentelemetry-helm-charts"
-    chart       = "opentelemetry-operator"
-    version     = "0.10.0"
-    namespace   = local.namespace
-    timeout     = "1200"
-    description = "ADOT Operator helm chart"
-    values      = []
-  }
-
-  helm_config = merge(
-    local.default_helm_config,
-    var.helm_config
-  )
-}

outputs.tf

@@ -12,10 +12,22 @@ output "eks_cluster_version" {
   value = data.aws_eks_cluster.eks_cluster.version
 }
 
-output "prometheus_endpoint" {
+output "managed_prometheus_workspace_endpoint" {
-  value = [aws_prometheus_workspace.this.*.prometheus_endpoint]
+  value = local.amp_ws_endpoint
 }
 
-output "prometheus_id" {
+output "managed_prometheus_workspace_id" {
-  value = [aws_prometheus_workspace.this.*.id]
+  value = local.amp_ws_id
+}
+
+output "managed_prometheus_workspace_region" {
+  value = local.amp_ws_region
+}
+
+output "managed_grafana_workspace_endpoint" {
+  value = local.amg_ws_endpoint
+}
+
+output "grafana_dashboards_folder_id" {
+  value = grafana_folder.this.id
 }

variables.tf

@@ -1,10 +1,3 @@
-variable "java" {
-  default = {
-    a = ""
-    b = ""
-  }
-}
-
 variable "eks_cluster_id" {
   description = "EKS Cluster Id"
   type        = string
@@ -48,13 +41,13 @@ variable "enable_managed_prometheus" {
   default = true
 }
 
-variable "managed_prometheus_id" {
+variable "managed_prometheus_workspace_id" {
   description = "AWS Managed Prometheus Workspace ID"
   type        = string
   default     = ""
 }
 
-variable "managed_prometheus_region" {
+variable "managed_prometheus_workspace_region" {
   description = "AWS Managed Prometheus Workspace Region"
   type        = string
   default     = null

workloads/infra/locals.tf

@@ -0,0 +1,31 @@
+data "aws_partition" "current" {}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_region" "current" {}
+
+data "aws_eks_cluster" "eks_cluster" {
+  name = var.eks_cluster_id
+}
+
+locals {
+  name      = "adot-collector-kubeprometheus"
+  namespace = try(var.config.helm_config.namespace, local.name)
+
+  eks_oidc_issuer_url  = replace(data.aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer, "https://", "")
+  eks_cluster_endpoint = data.aws_eks_cluster.eks_cluster.endpoint
+
+  context = {
+    aws_caller_identity_account_id = data.aws_caller_identity.current.account_id
+    aws_caller_identity_arn        = data.aws_caller_identity.current.arn
+    aws_eks_cluster_endpoint       = local.eks_cluster_endpoint
+    aws_partition_id               = data.aws_partition.current.partition
+    aws_region_name                = data.aws_region.current.name
+    eks_cluster_id                 = var.eks_cluster_id
+    eks_oidc_issuer_url            = local.eks_oidc_issuer_url
+    eks_oidc_provider_arn          = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.eks_oidc_issuer_url}"
+    tags                           = var.tags
+    irsa_iam_role_path             = var.irsa_iam_role_path
+    irsa_iam_permissions_boundary  = var.irsa_iam_permissions_boundary
+  }
+}

workloads/infra/main.tf

@@ -1,7 +1,3 @@
-locals {
-  name      = "adot-collector-kubeprometheus"
-  namespace = try(var.config.helm_config.namespace, local.name)
-}
 
 terraform {
   required_providers {
@@ -12,6 +8,11 @@ terraform {
   }
 }
 
+provider "grafana" {
+  url  = var.managed_grafana_workspace_endpoint
+  auth = var.grafana_api_key
+}
+
 resource "helm_release" "kube_state_metrics" {
   count            = var.config.enable_kube_state_metrics ? 1 : 0
   chart            = var.config.ksm_helm_chart_name
@@ -47,7 +48,6 @@ resource "helm_release" "prometheus_node_exporter" {
     }
   }
 }
-data "aws_partition" "current" {}
 
 module "helm_addon" {
   source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
@@ -66,11 +66,11 @@ module "helm_addon" {
   set_values = [
     {
       name  = "ampurl"
-      value = "${var.amp_endpoint}api/v1/remote_write"
+      value = "${var.managed_prometheus_workspace_endpoint}api/v1/remote_write"
     },
     {
       name  = "region"
-      value = var.amp_region
+      value = var.managed_prometheus_workspace_region
     },
     {
       name  = "prometheusMetricsEndpoint"
@@ -102,5 +102,5 @@ module "helm_addon" {
     irsa_iam_policies                 = ["arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess"]
   }
 
-  addon_context = var.addon_context
+  addon_context = local.context
 }

workloads/infra/variables.tf

@@ -1,48 +1,56 @@
+variable "eks_cluster_id" {
+  description = "EKS Cluster Id"
+  type        = string
+}
 
 
-# ADOT variable
 variable "helm_config" {
   description = "Helm Config for Prometheus"
   type        = any
   default     = {}
 }
 
-variable "amp_endpoint" {
+variable "irsa_iam_role_path" {
+  description = "IAM role path for IRSA roles"
+  type        = string
+  default     = "/"
+}
+
+variable "irsa_iam_permissions_boundary" {
+  description = "IAM permissions boundary for IRSA roles"
+  type        = string
+  default     = ""
+}
+
+variable "managed_prometheus_workspace_endpoint" {
   description = "Amazon Managed Prometheus Workspace Endpoint"
   type        = string
   default     = null
 }
-variable "amp_id" {
+variable "managed_prometheus_workspace_id" {
   description = "Amazon Managed Prometheus Workspace ID"
   type        = string
   default     = null
 }
 
-variable "amp_region" {
+variable "managed_prometheus_workspace_region" {
   description = "Amazon Managed Prometheus Workspace's Region"
   type        = string
   default     = null
 }
 
-variable "dashboards_folder_id" {
+variable "managed_grafana_workspace_endpoint" {
-  type = string
+  type    = string
+  default = ""
 }
 
-variable "addon_context" {
+variable "grafana_api_key" {
-  description = "Input configuration for the addon"
+  type    = string
-  type = object({
+  default = null
-    aws_caller_identity_account_id = string
+}
-    aws_caller_identity_arn        = string
+
-    aws_eks_cluster_endpoint       = string
+variable "dashboards_folder_id" {
-    aws_partition_id               = string
+  type = string
-    aws_region_name                = string
-    eks_cluster_id                 = string
-    eks_oidc_issuer_url            = string
-    eks_oidc_provider_arn          = string
-    irsa_iam_permissions_boundary  = string
-    irsa_iam_role_path             = string
-    tags                           = map(string)
-  })
 }
 
 variable "config" {
@@ -98,3 +106,9 @@ variable "config" {
   }
   nullable = false
 }
+
+variable "tags" {
+  description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)"
+  type        = map(string)
+  default     = {}
+}