K8s operator metrics (#185)

* Add kube admin metrics

* Update chart version

* Variable verbosity level for adot collector

modules/eks-monitoring/README.md

@@ -65,6 +65,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_adot_loglevel"></a> [adot\_loglevel](#input\_adot\_loglevel) | Verbosity level for ADOT collector logs | `string` | `"warn"` | no |
 | <a name="input_custom_metrics_config"></a> [custom\_metrics\_config](#input\_custom\_metrics\_config) | Configuration object to enable custom metrics collection | <pre>object({<br>    ports = list(number)<br>    # paths = optional(list(string), ["/metrics"])<br>    # list of samples to be dropped by label prefix, ex: go_ -> discards go_.*<br>    dropped_series_prefixes = list(string)<br>  })</pre> | <pre>{<br>  "dropped_series_prefixes": [<br>    "unspecified"<br>  ],<br>  "ports": []<br>}</pre> | no |
 | <a name="input_eks_cluster_id"></a> [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes |
 | <a name="input_enable_alerting_rules"></a> [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no |

modules/eks-monitoring/main.tf

@@ -105,6 +105,10 @@ module "helm_addon" {
       name  = "globalScrapeTimeout"
       value = var.prometheus_config.global_scrape_timeout
     },
+    {
+      name  = "adotLoglevel"
+      value = var.adot_loglevel
+    },
     {
       name  = "accountId"
       value = local.context.aws_caller_identity_account_id

modules/eks-monitoring/otel-config/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: opentelemetry
 description: A Helm chart to install otel operator
 type: application
-version: 0.5.0
+version: 0.6.0
-appVersion: 0.5.0
+appVersion: 0.6.0

modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml

@@ -83,6 +83,26 @@ spec:
                 regex: $K8S_NODE_NAME
                 source_labels: [__meta_kubernetes_node_name]
               {{ end }}
+            - job_name: 'kube-admin'
+              scheme: https
+              tls_config:
+                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+                insecure_skip_verify: true
+              bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+              kubernetes_sd_configs:
+              - role: node
+              relabel_configs:
+              - target_label: __address__
+                replacement: kubernetes.default.svc.cluster.local:443
+              {{ if .Values.enableTracing }}
+              - action: keep
+                regex: $K8S_NODE_NAME
+                source_labels: [__meta_kubernetes_node_name]
+              {{ end }}
+              metric_relabel_configs:
+              - action: keep
+                source_labels: [__name__]
+                regex: 'apiserver_(request_duration_seconds|storage_list_duration_seconds|admission_controller_admission_duration_seconds|flowcontrol_request_wait_duration_seconds).*|apiserver_(admission_webhook_fail_open_count|tls_handshake_errors_total|request_total)|rest_client_request_duration_seconds.*|rest_client_requests_total|etcd_(request_duration_seconds|db_total_size_in_bytes).*'
             - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0
               honor_timestamps: true
               scrape_interval: {{ .Values.globalScrapeInterval }}
@@ -1100,117 +1120,6 @@ spec:
                     own_namespace: false
                     names:
                     - kube-system
-            - job_name: serviceMonitor/default/kube-prometheus-stack-kube-etcd/0
-              honor_timestamps: true
-              scrape_interval: {{ .Values.globalScrapeInterval }}
-              scrape_timeout: {{ .Values.globalScrapeTimeout }}
-              scheme: http
-              authorization:
-                type: Bearer
-                credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-              follow_redirects: true
-              enable_http2: true
-              relabel_configs:
-              - source_labels: [job]
-                separator: ;
-                regex: (.*)
-                target_label: __tmp_prometheus_job_name
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app]
-                separator: ;
-                regex: (kube-prometheus-stack-kube-etcd);true
-                replacement: $$1
-                action: keep
-              - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release]
-                separator: ;
-                regex: (kube-prometheus-stack);true
-                replacement: $$1
-                action: keep
-              - source_labels: [__meta_kubernetes_endpoint_port_name]
-                separator: ;
-                regex: http-metrics
-                replacement: $$1
-                action: keep
-              - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name]
-                separator: ;
-                regex: Node;(.*)
-                target_label: node
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name]
-                separator: ;
-                regex: Pod;(.*)
-                target_label: pod
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_namespace]
-                separator: ;
-                regex: (.*)
-                target_label: namespace
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_name]
-                separator: ;
-                regex: (.*)
-                target_label: service
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_pod_name]
-                separator: ;
-                regex: (.*)
-                target_label: pod
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_pod_container_name]
-                separator: ;
-                regex: (.*)
-                target_label: container
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_name]
-                separator: ;
-                regex: (.*)
-                target_label: job
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_label_jobLabel]
-                separator: ;
-                regex: (.+)
-                target_label: job
-                replacement: $$1
-                action: replace
-              - separator: ;
-                regex: (.*)
-                target_label: endpoint
-                replacement: http-metrics
-                action: replace
-              - source_labels: [__address__]
-                separator: ;
-                regex: (.*)
-                modulus: 1
-                target_label: __tmp_hash
-                replacement: $$1
-                action: hashmod
-              - source_labels: [__tmp_hash]
-                separator: ;
-                regex: "0"
-                replacement: $$1
-                action: keep
-              {{ if .Values.enableTracing }}
-              - action: keep
-                regex: $K8S_NODE_NAME
-                source_labels: [__meta_kubernetes_endpoint_node_name]
-              {{ end }}
-              kubernetes_sd_configs:
-                - role: endpoints
-                  kubeconfig_file: ""
-                  follow_redirects: true
-                  enable_http2: true
-                  namespaces:
-                    own_namespace: false
-                    names:
-                    - kube-system
             - job_name: serviceMonitor/default/kube-prometheus-stack-kube-controller-manager/0
               honor_timestamps: true
               scrape_interval: {{ .Values.globalScrapeInterval }}
@@ -1431,115 +1340,6 @@ spec:
                     own_namespace: false
                     names:
                     - kube-system
-            - job_name: serviceMonitor/default/kube-prometheus-stack-apiserver/0
-              honor_timestamps: true
-              scrape_interval: {{ .Values.globalScrapeInterval }}
-              scrape_timeout: {{ .Values.globalScrapeTimeout }}
-              scheme: https
-              authorization:
-                type: Bearer
-                credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-              tls_config:
-                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-                server_name: kubernetes
-              follow_redirects: true
-              enable_http2: true
-              relabel_configs:
-              - source_labels: [job]
-                separator: ;
-                regex: (.*)
-                target_label: __tmp_prometheus_job_name
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_label_component, __meta_kubernetes_service_labelpresent_component]
-                separator: ;
-                regex: (kubernetes);true
-                replacement: $$1
-                action: keep
-              - source_labels: [__meta_kubernetes_endpoint_port_name]
-                separator: ;
-                regex: https
-                replacement: $$1
-                action: keep
-              - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name]
-                separator: ;
-                regex: Node;(.*)
-                target_label: node
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name]
-                separator: ;
-                regex: Pod;(.*)
-                target_label: pod
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_namespace]
-                separator: ;
-                regex: (.*)
-                target_label: namespace
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_name]
-                separator: ;
-                regex: (.*)
-                target_label: service
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_pod_name]
-                separator: ;
-                regex: (.*)
-                target_label: pod
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_pod_container_name]
-                separator: ;
-                regex: (.*)
-                target_label: container
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_name]
-                separator: ;
-                regex: (.*)
-                target_label: job
-                replacement: $$1
-                action: replace
-              - source_labels: [__meta_kubernetes_service_label_component]
-                separator: ;
-                regex: (.+)
-                target_label: job
-                replacement: $$1
-                action: replace
-              - separator: ;
-                regex: (.*)
-                target_label: endpoint
-                replacement: https
-                action: replace
-              - source_labels: [__address__]
-                separator: ;
-                regex: (.*)
-                modulus: 1
-                target_label: __tmp_hash
-                replacement: $$1
-                action: hashmod
-              - source_labels: [__tmp_hash]
-                separator: ;
-                regex: "0"
-                replacement: $$1
-                action: keep
-              {{ if .Values.enableTracing }}
-              - action: keep
-                regex: $K8S_NODE_NAME
-                source_labels: [__meta_kubernetes_endpoint_node_name]
-              {{ end }}
-              kubernetes_sd_configs:
-                - role: endpoints
-                  kubeconfig_file: ""
-                  follow_redirects: true
-                  enable_http2: true
-                  namespaces:
-                    own_namespace: false
-                    names:
-                    - default
             - job_name: serviceMonitor/default/kube-prometheus-stack-alertmanager/0
               honor_timestamps: true
               scrape_interval: {{ .Values.globalScrapeInterval }}
@@ -1776,7 +1576,7 @@ spec:
         resource_to_telemetry_conversion:
           enabled: true
       logging:
-        loglevel: warn
+        loglevel: {{ .Values.adotLoglevel }}
     extensions:
       sigv4auth:
         region: {{ .Values.region }}

modules/eks-monitoring/otel-config/values.yaml

@@ -23,3 +23,5 @@ javaPrometheusMetricsEndpoint: ${java_prometheus_metrics_endpoint}
 enableNginx: ${enable_nginx}
 nginxScrapeSampleLimit: ${nginx_scrape_sample_limit}
 nginxPrometheusMetricsEndpoint: ${nginx_prometheus_metrics_endpoint}
+
+adotLoglevel: ${adot_loglevel}

modules/eks-monitoring/variables.tf

@@ -33,6 +33,12 @@ variable "irsa_iam_permissions_boundary" {
   default     = null
 }
 
+variable "adot_loglevel" {
+  description = "Verbosity level for ADOT collector logs"
+  type        = string
+  default     = "warn"
+}
+
 variable "managed_prometheus_workspace_endpoint" {
   description = "Amazon Managed Prometheus Workspace Endpoint"
   type        = string