Add kubeprom collector

examples/main.tf

@@ -24,11 +24,6 @@ provider "helm" {
   }
 }
 
-# provider "grafana" {
-#   url  = var.grafana_endpoint
-#   auth = var.grafana_api_key
-# }
-
 terraform {
   required_providers {
     grafana = {

examples/workloads.tf

@@ -7,8 +7,6 @@ module "eks_observability_accelerator" {
   aws_region     = var.aws_region
   eks_cluster_id = var.eks_cluster_id
 
-  # TODO: create also a cluster, VPC -- check if enough VPCs
-
   # deploys AWS Distro for OpenTelemetry operator into the cluster
   enable_amazon_eks_adot = true
 
@@ -16,7 +14,7 @@ module "eks_observability_accelerator" {
   enable_cert_manager = true
 
   # # -- or enable opentelemetry operator
-  enable_opentelemetry_operator = false #-- true doesn't work for me, needs fix
+  enable_opentelemetry_operator = false
   #open_telemetry_operator_config = map() // custom config
 
   # creates a new AMP workspace, defaults to true
@@ -34,11 +32,9 @@ module "eks_observability_accelerator" {
   managed_grafana_workspace_id = var.managed_grafana_workspace_id
   grafana_api_key              = var.grafana_api_key
 
-
-
-  enable_java                 = true
-  enable_java_recording_rules = true
-
+  # enable workload-specific collector, metrics, alerts and dashboards
+  enable_java                 = false
+  enable_java_recording_rules = false
 
   # enable_haproxy = true
   # haproxy_config = {
@@ -46,29 +42,10 @@ module "eks_observability_accelerator" {
   #   grafana_endpoint = module.grafana.endpoint
   # }
 
-
-  # java_config = {
-  #   amp_endpoint     = ""
-  #   grafana_endpoint = ""
-  # }
-
-  # # -- or provide custom alerts definition
-  # prometheus_custom_alert_rule = var.prometheus_custom_alert_rule
-
-
-  # # create grafana workspace, and customer to deal with authentication later
-  # create_managed_grafana_workspace = true
-  # grafana_auth_provider            = var.grafana_auth_provider       //SAML or AWS_SSO
-  # grafana_account_access_type      = var.grafana_account_access_type // CURRENT_ACCOUNT or ORGANIZATION
-  # grafana_permission_type          = var.grafana_permission_type     // SERVICE_MANAGED or CUSTOMER_MANAGED
-  # grafana_permission_role_arn      = var.grafana_permission_role_arn // if CUSTOMER_MANAGED
-
-  # # -- or using existing amg workspace. so we can use API for keys
-
+  enable_infra_metrics = true
+  #infra_metrics_config = {}
 
   tags = local.tags
-
-
 }
 # module "amp" {
 

main.tf

@@ -81,3 +81,41 @@ module "java" {
     module.operator
   ]
 }
+
+module "infra" {
+  count  = var.enable_infra_metrics ? 1 : 0
+  source = "./modules/workloads/kube-prometheus"
+
+  addon_context = local.context
+
+  amp_endpoint = local.amp_ws_endpoint
+  amp_id       = local.amp_ws_id
+  amp_region   = local.amp_ws_region
+
+  config = var.infra_metrics_config
+
+
+  # enable_kube_state_metrics = var.infra_metrics_config.enable_kube_state_metrics
+  # kms_create_namespace      = var.infra_metrics_config.kms_create_namespace
+  # ksm_k8s_namespace         = var.infra_metrics_config.ksm_k8s_namespace
+  # ksm_helm_chart_name       = var.infra_metrics_config.ksm_helm_chart_name
+  # ksm_helm_chart_version    = var.infra_metrics_config.ksm_helm_chart_version
+  # ksm_helm_release_name     = var.infra_metrics_config.ksm_helm_release_name
+  # ksm_helm_repo_url         = var.infra_metrics_config.ksm_helm_repo_url
+  # ksm_helm_settings         = var.infra_metrics_config.ksm_helm_settings
+  # ksm_helm_values           = var.infra_metrics_config.ksm_helm_values
+
+  # enable_node_exporter  = var.infra_metrics_config.enable_node_exporter
+  # ne_create_namespace   = var.infra_metrics_config.ne_create_namespace
+  # ne_helm_chart_name    = var.infra_metrics_config.ne_helm_chart_name
+  # ne_helm_chart_version = var.infra_metrics_config.ne_helm_chart_version
+  # ne_helm_release_name  = var.infra_metrics_config.ne_helm_release_name
+  # ne_helm_repo_url      = var.infra_metrics_config.ne_helm_repo_url
+  # ne_helm_settings      = var.infra_metrics_config.ksm_helm_settings
+  # ne_helm_values        = var.infra_metrics_config.ksm_helm_values
+
+  depends_on = [
+    module.operator
+  ]
+
+}

modules/workloads/kube-prometheus/README.md

@@ -0,0 +1,49 @@
+# Observability Pattern for Java/JMX
+
+This module provides an automated experience around Observability for Nginx workloads.
+It provides the following resources:
+
+- AWS Distro For OpenTelemetry Operator and Collector
+- AWS Managed Grafana Dashboard and data source
+- Alerts and recording rules with AWS Managed Service for Prometheus
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_helm_addon"></a> [helm\_addon](#module\_helm\_addon) | ../helm-addon | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br>    aws_caller_identity_account_id = string<br>    aws_caller_identity_arn        = string<br>    aws_eks_cluster_endpoint       = string<br>    aws_partition_id               = string<br>    aws_region_name                = string<br>    eks_cluster_id                 = string<br>    eks_oidc_issuer_url            = string<br>    eks_oidc_provider_arn          = string<br>    irsa_iam_permissions_boundary  = string<br>    irsa_iam_role_path             = string<br>    tags                           = map(string)<br>  })</pre> | n/a | yes |
+| <a name="input_amazon_prometheus_workspace_endpoint"></a> [amazon\_prometheus\_workspace\_endpoint](#input\_amazon\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `null` | no |
+| <a name="input_amazon_prometheus_workspace_region"></a> [amazon\_prometheus\_workspace\_region](#input\_amazon\_prometheus\_workspace\_region) | Amazon Managed Prometheus Workspace's Region | `string` | `null` | no |
+| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/workloads/kube-prometheus/main.tf

@@ -0,0 +1,104 @@
+locals {
+  name      = "adot-collector-kubeprometheus"
+  namespace = try(var.config.helm_config.namespace, local.name)
+}
+resource "helm_release" "kube_state_metrics" {
+  count            = var.config.enable_kube_state_metrics ? 1 : 0
+  chart            = var.config.ksm_helm_chart_name
+  create_namespace = var.config.kms_create_namespace
+  namespace        = var.config.ksm_k8s_namespace
+  name             = var.config.ksm_helm_release_name
+  version          = var.config.ksm_helm_chart_version
+  repository       = var.config.ksm_helm_repo_url
+
+  # values = [
+  #   var.config.ksm_helm_values
+  # ]
+
+  dynamic "set" {
+    for_each = var.config.ksm_helm_settings
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+}
+
+resource "helm_release" "prometheus_node_exporter" {
+  count            = var.config.enable_node_exporter ? 1 : 0
+  chart            = var.config.ne_helm_chart_name
+  create_namespace = var.config.ne_create_namespace
+  namespace        = var.config.ne_k8s_namespace
+  name             = var.config.ne_helm_release_name
+  version          = var.config.ne_helm_chart_version
+  repository       = var.config.ne_helm_repo_url
+
+  # values = [
+  #   var.config.ne_helm_values
+  # ]
+
+  dynamic "set" {
+    for_each = var.config.ne_helm_settings
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+}
+data "aws_partition" "current" {}
+
+module "helm_addon" {
+  source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
+
+  helm_config = merge(
+    {
+      name        = local.name
+      chart       = "${path.module}/otel-config"
+      version     = "0.2.0"
+      namespace   = local.namespace
+      description = "ADOT helm Chart deployment configuration"
+    },
+    var.helm_config
+  )
+
+  set_values = [
+    {
+      name  = "ampurl"
+      value = "${var.amp_endpoint}api/v1/remote_write"
+    },
+    {
+      name  = "region"
+      value = var.amp_region
+    },
+    {
+      name  = "prometheusMetricsEndpoint"
+      value = "metrics"
+    },
+    {
+      name  = "prometheusMetricsPort"
+      value = 8888
+    },
+    {
+      name  = "scrapeInterval"
+      value = "15s"
+    },
+    {
+      name  = "scrapeTimeout"
+      value = "10s"
+    },
+    {
+      name  = "scrapeSampleLimit"
+      value = 1000
+    }
+  ]
+
+  irsa_config = {
+    create_kubernetes_namespace       = true
+    kubernetes_namespace              = local.namespace
+    create_kubernetes_service_account = true
+    kubernetes_service_account        = try(var.config.helm_config.service_account, local.name)
+    irsa_iam_policies                 = ["arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess"]
+  }
+
+  addon_context = var.addon_context
+}

modules/workloads/kube-prometheus/otel-config/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: opentelemetry
+description: A Helm chart to install otel operator
+type: application
+version: 0.2.0
+appVersion: v0.1.0

modules/workloads/kube-prometheus/otel-config/templates/clusterrole.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: otel-prometheus-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - nonResourceURLs:
+      - /metrics
+    verbs:
+      - get

modules/workloads/kube-prometheus/otel-config/templates/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: otel-prometheus-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: otel-prometheus-role
+subjects:
+  - kind: ServiceAccount
+    name: adot-collector-kubeprometheus
+    namespace: adot-collector-kubeprometheus

modules/workloads/kube-prometheus/otel-config/values.yaml

@@ -0,0 +1,7 @@
+ampurl: ${amp_url}
+region: ${region}
+prometheusMetricsEndpoint: ${prometheus_metrics_endpoint}
+prometheusMetricsPort: ${prometheus_metrics_port}
+scrapeInterval: ${scrape_interval}
+scrapeTimeout: ${scrape_timeout}
+scrapeSampleLimit: ${scrape_sample_limit}

modules/workloads/kube-prometheus/variables.tf

@@ -0,0 +1,198 @@
+
+
+# ADOT variable
+variable "helm_config" {
+  description = "Helm Config for Prometheus"
+  type        = any
+  default     = {}
+}
+
+variable "amp_endpoint" {
+  description = "Amazon Managed Prometheus Workspace Endpoint"
+  type        = string
+  default     = null
+}
+variable "amp_id" {
+  description = "Amazon Managed Prometheus Workspace ID"
+  type        = string
+  default     = null
+}
+
+variable "amp_region" {
+  description = "Amazon Managed Prometheus Workspace's Region"
+  type        = string
+  default     = null
+}
+
+variable "addon_context" {
+  description = "Input configuration for the addon"
+  type = object({
+    aws_caller_identity_account_id = string
+    aws_caller_identity_arn        = string
+    aws_eks_cluster_endpoint       = string
+    aws_partition_id               = string
+    aws_region_name                = string
+    eks_cluster_id                 = string
+    eks_oidc_issuer_url            = string
+    eks_oidc_provider_arn          = string
+    irsa_iam_permissions_boundary  = string
+    irsa_iam_role_path             = string
+    tags                           = map(string)
+  })
+}
+
+variable "config" {
+  type = object({
+    helm_config = map(any)
+
+    enable_kube_state_metrics = bool
+    kms_create_namespace      = bool
+    ksm_k8s_namespace         = string
+    ksm_helm_chart_name       = string
+    ksm_helm_chart_version    = string
+    ksm_helm_release_name     = string
+    ksm_helm_repo_url         = string
+    ksm_helm_settings         = map(string)
+    ksm_helm_values           = map(any)
+
+    enable_node_exporter  = bool
+    ne_create_namespace   = bool
+    ne_k8s_namespace      = string
+    ne_helm_chart_name    = string
+    ne_helm_chart_version = string
+    ne_helm_release_name  = string
+    ne_helm_repo_url      = string
+    ne_helm_settings      = map(string)
+    ne_helm_values        = map(any)
+  })
+
+  default = {
+    enable_kube_state_metrics = true
+    enable_node_exporter      = true
+    helm_config               = {}
+
+    kms_create_namespace   = true
+    ksm_helm_chart_name    = "kube-state-metrics"
+    ksm_helm_chart_version = "4.9.2"
+    ksm_helm_release_name  = "kube-state-metrics"
+    ksm_helm_repo_url      = "https://prometheus-community.github.io/helm-charts"
+    ksm_helm_settings      = {}
+    ksm_helm_values        = {}
+    ksm_k8s_namespace      = "kube-system"
+
+    ne_create_namespace   = true
+    ne_k8s_namespace      = "prometheus-node-exporter"
+    ne_helm_chart_name    = "prometheus-node-exporter"
+    ne_helm_chart_version = "2.0.3"
+    ne_helm_release_name  = "prometheus-node-exporter"
+    ne_helm_repo_url      = "https://prometheus-community.github.io/helm-charts"
+    ne_helm_settings      = {}
+    ne_helm_values        = {}
+  }
+  nullable = false
+}
+
+# # Kube-state-metrics Variables
+# variable "enable_kube_state_metrics" {
+#   description = "Variable indicating whether deployment is enabled"
+#   type        = bool
+#   default     = true
+# }
+
+# # Helm
+
+# variable "helm_create_namespace_ksm" {
+#   type        = bool
+#   default     = true
+#   description = "Create the namespace if it does not yet exist"
+# }
+
+# variable "helm_chart_name_ksm" {
+#   type        = string
+#   default     = "kube-state-metrics"
+#   description = "Helm chart name to be installed"
+# }
+
+# variable "helm_chart_version_ksm" {
+#   type        = string
+#   default     = "4.9.2"
+#   description = "Version of the Helm chart"
+# }
+
+# variable "helm_release_name_ksm" {
+#   type        = string
+#   default     = "kube-state-metrics"
+#   description = "Helm release name"
+# }
+
+# variable "helm_repo_url_ksm" {
+#   type        = string
+#   default     = "https://prometheus-community.github.io/helm-charts"
+#   description = "Helm repository"
+# }
+
+# # K8s
+# variable "k8s_namespace_ksm" {
+#   type        = string
+#   default     = "kube-system"
+#   description = "The K8s namespace in which the prometheus-node-exporter service account has been created"
+# }
+
+# variable "settings" {
+#   type        = map(any)
+#   default     = {}
+#   description = "Additional settings which will be passed to the Helm chart values, see https://hub.helm.sh/charts/stable/prometheus-node-exporter"
+# }
+
+# variable "values" {
+#   type        = string
+#   default     = ""
+#   description = "Additional yaml encoded values which will be passed to the Helm chart."
+# }
+
+# # Node exporter Variables
+
+# variable "enabled_node_exporter" {
+#   description = "Variable indicating whether deployment is enabled"
+#   type        = bool
+#   default     = true
+# }
+
+# # Helm
+
+# variable "helm_create_namespace_ne" {
+#   type        = bool
+#   default     = true
+#   description = "Create the namespace if it does not yet exist"
+# }
+
+# variable "helm_chart_name_ne" {
+#   type        = string
+#   default     = "prometheus-node-exporter"
+#   description = "Helm chart name to be installed"
+# }
+
+# variable "helm_chart_version_ne" {
+#   type        = string
+#   default     = "2.0.3"
+#   description = "Version of the Helm chart"
+# }
+
+# variable "helm_release_name_ne" {
+#   type        = string
+#   default     = "prometheus-node-exporter"
+#   description = "Helm release name"
+# }
+
+# variable "helm_repo_url_ne" {
+#   type        = string
+#   default     = "https://prometheus-community.github.io/helm-charts"
+#   description = "Helm repository"
+# }
+
+# # K8s
+# variable "helm_repo_url_ne" {
+#   type        = string
+#   default     = "kube-system"
+#   description = "The K8s namespace in which the prometheus-node-exporter service account has been created"
+# }

modules/workloads/kube-prometheus/versions.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_version = ">= 1.0.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 3.72"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.10"
+    }
+  }
+}

variables.tf

@@ -111,3 +111,61 @@ variable "tags" {
   type        = map(string)
   default     = {}
 }
+
+
+variable "enable_infra_metrics" {
+  type    = bool
+  default = false
+}
+
+variable "infra_metrics_config" {
+  type = object({
+    helm_config = map(any)
+
+    enable_kube_state_metrics = bool
+    kms_create_namespace      = bool
+    ksm_k8s_namespace         = string
+    ksm_helm_chart_name       = string
+    ksm_helm_chart_version    = string
+    ksm_helm_release_name     = string
+    ksm_helm_repo_url         = string
+    ksm_helm_settings         = map(string)
+    ksm_helm_values           = map(any)
+
+    enable_node_exporter  = bool
+    ne_create_namespace   = bool
+    ne_k8s_namespace      = string
+    ne_helm_chart_name    = string
+    ne_helm_chart_version = string
+    ne_helm_release_name  = string
+    ne_helm_repo_url      = string
+    ne_helm_settings      = map(string)
+    ne_helm_values        = map(any)
+  })
+
+  default = null
+
+  # default = {
+  #   enable_kube_state_metrics = true
+  #   enable_node_exporter      = true
+  #   helm_config               = {}
+
+  #   kms_create_namespace   = true
+  #   ksm_helm_chart_name    = "kube-state-metrics"
+  #   ksm_helm_chart_version = "4.9.2"
+  #   ksm_helm_release_name  = "kube-state-metrics"
+  #   ksm_helm_repo_url      = "https://prometheus-community.github.io/helm-charts"
+  #   ksm_helm_settings      = {}
+  #   ksm_helm_values        = {}
+  #   ksm_k8s_namespace      = "kube-system"
+
+  #   ne_create_namespace   = true
+  #   ne_k8s_namespace      = "prometheus-node-exporter"
+  #   ne_helm_chart_name    = "prometheus-node-exporter"
+  #   ne_helm_chart_version = "2.0.3"
+  #   ne_helm_release_name  = "prometheus-node-exporter"
+  #   ne_helm_repo_url      = "https://prometheus-community.github.io/helm-charts"
+  #   ne_helm_settings      = {}
+  #   ne_helm_values        = {}
+  # }
+}