From 6ca939d9232b87b2d4a8e3bcf22fcdf159551d38 Mon Sep 17 00:00:00 2001
From: Reshma Mathews <114662591+RAMathews@users.noreply.github.com>
Date: Thu, 17 Aug 2023 15:50:34 +0100
Subject: [PATCH] Adot health monitoring (#212)

* Added adot health metric collection

* corrected alignment related error in Opentelemetry config

* reverted the links

* pre-commit

* Update dashboards.tf

split adothealth dashboard

* split dashboards

* changed dashboard link

* corrected flux_kustomization_path

* Pre-commit
---
 modules/eks-monitoring/README.md              |  3 +++
 modules/eks-monitoring/dashboards.tf          | 26 +++++++++++++++++++
 modules/eks-monitoring/locals.tf              | 13 ++++++++++
 modules/eks-monitoring/main.tf                |  5 ++++
 .../templates/opentelemetrycollector.yaml     | 21 +++++++++++++++
 .../eks-monitoring/otel-config/values.yaml    |  2 ++
 modules/eks-monitoring/variables.tf           | 24 +++++++++++++++++
 7 files changed, 94 insertions(+)

diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md
index eed5281..9817be9 100644
--- a/modules/eks-monitoring/README.md
+++ b/modules/eks-monitoring/README.md
@@ -55,6 +55,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this
 | [helm_release.grafana_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.prometheus_node_exporter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [kubectl_manifest.adothealth_monitoring_dashboards](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
 | [kubectl_manifest.api_server_dashboards](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
 | [kubectl_manifest.flux_gitrepository](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
 | [kubectl_manifest.flux_kustomization](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
@@ -68,9 +69,11 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_adot_loglevel"></a> [adot\_loglevel](#input\_adot\_loglevel) | Verbosity level for ADOT collector logs. This accepts (detailed\|normal\|basic), see https://aws-otel.github.io/docs/components/misc-exporters for mor infos. | `string` | `"normal"` | no |
+| <a name="input_adothealth_monitoring_config"></a> [adothealth\_monitoring\_config](#input\_adothealth\_monitoring\_config) | Config object for API server monitoring | <pre>object({<br>    flux_gitrepository_name   = string<br>    flux_gitrepository_url    = string<br>    flux_gitrepository_branch = string<br>    flux_kustomization_name   = string<br>    flux_kustomization_path   = string<br><br>    dashboards = object({<br>      grafana_adothealth_dashboard_url = string<br>    })<br>  })</pre> | `null` | no |
 | <a name="input_apiserver_monitoring_config"></a> [apiserver\_monitoring\_config](#input\_apiserver\_monitoring\_config) | Config object for API server monitoring | <pre>object({<br>    flux_gitrepository_name   = string<br>    flux_gitrepository_url    = string<br>    flux_gitrepository_branch = string<br>    flux_kustomization_name   = string<br>    flux_kustomization_path   = string<br><br>    dashboards = object({<br>      basic           = string<br>      advanced        = string<br>      troubleshooting = string<br>    })<br>  })</pre> | `null` | no |
 | <a name="input_custom_metrics_config"></a> [custom\_metrics\_config](#input\_custom\_metrics\_config) | Configuration object to enable custom metrics collection | <pre>map(object({<br>    enableBasicAuth       = bool<br>    path                  = string<br>    basicAuthUsername     = string<br>    basicAuthPassword     = string<br>    ports                 = string<br>    droppedSeriesPrefixes = string<br>  }))</pre> | `null` | no |
 | <a name="input_eks_cluster_id"></a> [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes |
+| <a name="input_enable_adotcollector_metrics"></a> [enable\_adotcollector\_metrics](#input\_enable\_adotcollector\_metrics) | Enables collection of ADOT collector metrics | `bool` | `true` | no |
 | <a name="input_enable_alerting_rules"></a> [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no |
 | <a name="input_enable_amazon_eks_adot"></a> [enable\_amazon\_eks\_adot](#input\_enable\_amazon\_eks\_adot) | Enables the ADOT Operator on the EKS Cluster | `bool` | `true` | no |
 | <a name="input_enable_apiserver_monitoring"></a> [enable\_apiserver\_monitoring](#input\_enable\_apiserver\_monitoring) | Enable EKS kube-apiserver monitoring, alerting and dashboards | `bool` | `true` | no |
diff --git a/modules/eks-monitoring/dashboards.tf b/modules/eks-monitoring/dashboards.tf
index 672ea1c..a427bdc 100644
--- a/modules/eks-monitoring/dashboards.tf
+++ b/modules/eks-monitoring/dashboards.tf
@@ -74,3 +74,29 @@ YAML
   count      = var.enable_apiserver_monitoring ? 1 : 0
   depends_on = [module.external_secrets]
 }
+
+# adot health dashboards
+resource "kubectl_manifest" "adothealth_monitoring_dashboards" {
+  yaml_body  = <<YAML
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: ${local.adothealth_monitoring_config.flux_kustomization_name}
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  path: ${local.adothealth_monitoring_config.flux_kustomization_path}
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: ${local.adothealth_monitoring_config.flux_gitrepository_name}
+  postBuild:
+    substitute:
+      AMG_AWS_REGION: ${var.managed_prometheus_workspace_region}
+      AMP_ENDPOINT_URL: ${var.managed_prometheus_workspace_endpoint}
+      AMG_ENDPOINT_URL: ${var.grafana_url}
+      GRAFANA_ADOTHEALTH_DASH_URL: ${local.adothealth_monitoring_config.dashboards.grafana_adothealth_dashboard_url}
+YAML
+  count      = var.enable_adotcollector_metrics ? 1 : 0
+  depends_on = [module.external_secrets]
+}
diff --git a/modules/eks-monitoring/locals.tf b/modules/eks-monitoring/locals.tf
index e7af94c..059dc41 100644
--- a/modules/eks-monitoring/locals.tf
+++ b/modules/eks-monitoring/locals.tf
@@ -119,4 +119,17 @@ locals {
       troubleshooting = "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/apiserver/apiserver-troubleshooting.json"
     }
   }
+
+  adothealth_monitoring_config = {
+    # can be overriden by providing a config
+    flux_gitrepository_name   = "aws-observability-accelerator"
+    flux_gitrepository_url    = "https://github.com/aws-observability/aws-observability-accelerator"
+    flux_gitrepository_branch = "main"
+    flux_kustomization_name   = "grafana-dashboards-adothealth"
+    flux_kustomization_path   = "./artifacts/grafana-operator-manifests/eks/adot"
+
+    dashboards = {
+      grafana_adothealth_dashboard_url = "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/adot/adothealth.json"
+    }
+  }
 }
diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf
index f0bb92f..a1fd63b 100644
--- a/modules/eks-monitoring/main.tf
+++ b/modules/eks-monitoring/main.tf
@@ -180,7 +180,12 @@ module "helm_addon" {
     {
       name  = "istioPrometheusMetricsEndpoint"
       value = try(var.istio_config.prometheus_metrics_endpoint, local.istio_pattern_config.prometheus_metrics_endpoint)
+    },
+    {
+      name  = "enableAdotcollectorMetrics"
+      value = var.enable_adotcollector_metrics
     }
+
   ]
 
   irsa_config = {
diff --git a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml
index 753d263..9ea4aed 100644
--- a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml
+++ b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml
@@ -26,6 +26,15 @@ spec:
             endpoint: {{ .Values.otlpGrpcEndpoint }}
           http:
             endpoint: {{ .Values.otlpHttpEndpoint }}
+      {{ if .Values.enableAdotcollectorMetrics }}
+      prometheus/1:
+        config:
+          scrape_configs:
+          - job_name: otel-collector-metrics
+            scrape_interval: 10s
+            static_configs:
+              - targets: ['localhost:8888']
+      {{ end }}
       prometheus:
         config:
           global:
@@ -1607,9 +1616,21 @@ spec:
           receivers: [prometheus, otlp]
           processors: [batch/metrics, attributes/metrics]
           exporters: [logging, prometheusremotewrite]
+        {{ if .Values.enableAdotcollectorMetrics }}
+        metrics/1:
+         receivers: [prometheus/1]
+         processors: []
+         exporters: [prometheusremotewrite]
+        {{ end }}
         {{ if .Values.enableTracing }}
         traces:
           receivers: [otlp]
           processors: [batch/traces]
           exporters: [logging, awsxray]
         {{ end }}
+      {{ if .Values.enableAdotcollectorMetrics }}
+      telemetry:
+        metrics:
+          address: 0.0.0.0:8888
+          level: basic
+      {{ end }}
diff --git a/modules/eks-monitoring/otel-config/values.yaml b/modules/eks-monitoring/otel-config/values.yaml
index 2e48954..5d74367 100644
--- a/modules/eks-monitoring/otel-config/values.yaml
+++ b/modules/eks-monitoring/otel-config/values.yaml
@@ -30,3 +30,5 @@ istioScrapeSampleLimit: ${istio_scrape_sample_limit}
 istioPrometheusMetricsEndpoint: ${istio_prometheus_metrics_endpoint}
 
 adotLoglevel: ${adot_loglevel}
+
+enableAdotcollectorMetrics: ${enable_adotcollector_metrics}
diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf
index da811de..531ca18 100644
--- a/modules/eks-monitoring/variables.tf
+++ b/modules/eks-monitoring/variables.tf
@@ -507,3 +507,27 @@ variable "target_secret_namespace" {
   type        = string
   default     = "grafana-operator"
 }
+
+variable "enable_adotcollector_metrics" {
+  description = "Enables collection of ADOT collector metrics"
+  type        = bool
+  default     = true
+}
+
+variable "adothealth_monitoring_config" {
+  description = "Config object for API server monitoring"
+  type = object({
+    flux_gitrepository_name   = string
+    flux_gitrepository_url    = string
+    flux_gitrepository_branch = string
+    flux_kustomization_name   = string
+    flux_kustomization_path   = string
+
+    dashboards = object({
+      grafana_adothealth_dashboard_url = string
+    })
+  })
+
+  # defaults are pre-computed in locals.tf, provide a full definition to override
+  default = null
+}