From a8fc12f7b7d7f4ed814d48061cf3e0eb4a646c9e Mon Sep 17 00:00:00 2001 From: Rodrigue Koffi Date: Thu, 4 Aug 2022 14:37:46 +0200 Subject: [PATCH] Add kubeprom collector --- examples/main.tf | 5 - examples/workloads.tf | 35 +- main.tf | 38 + modules/workloads/kube-prometheus/README.md | 49 + modules/workloads/kube-prometheus/main.tf | 104 ++ .../kube-prometheus/otel-config/Chart.yaml | 6 + .../otel-config/templates/clusterrole.yaml | 29 + .../templates/clusterrolebinding.yaml | 12 + .../templates/opentelemetrycollector.yaml | 1593 +++++++++++++++++ .../kube-prometheus/otel-config/values.yaml | 7 + modules/workloads/kube-prometheus/outputs.tf | 0 .../workloads/kube-prometheus/variables.tf | 198 ++ modules/workloads/kube-prometheus/versions.tf | 14 + variables.tf | 58 + 14 files changed, 2114 insertions(+), 34 deletions(-) create mode 100644 modules/workloads/kube-prometheus/README.md create mode 100644 modules/workloads/kube-prometheus/main.tf create mode 100644 modules/workloads/kube-prometheus/otel-config/Chart.yaml create mode 100644 modules/workloads/kube-prometheus/otel-config/templates/clusterrole.yaml create mode 100644 modules/workloads/kube-prometheus/otel-config/templates/clusterrolebinding.yaml create mode 100644 modules/workloads/kube-prometheus/otel-config/templates/opentelemetrycollector.yaml create mode 100644 modules/workloads/kube-prometheus/otel-config/values.yaml create mode 100644 modules/workloads/kube-prometheus/outputs.tf create mode 100644 modules/workloads/kube-prometheus/variables.tf create mode 100644 modules/workloads/kube-prometheus/versions.tf diff --git a/examples/main.tf b/examples/main.tf index d0d0980..8936626 100644 --- a/examples/main.tf +++ b/examples/main.tf @@ -24,11 +24,6 @@ provider "helm" { } } -# provider "grafana" { -# url = var.grafana_endpoint -# auth = var.grafana_api_key -# } - terraform { required_providers { grafana = { diff --git a/examples/workloads.tf b/examples/workloads.tf index 51b75ff..8336ba0 100644 --- a/examples/workloads.tf +++ b/examples/workloads.tf @@ -7,8 +7,6 @@ module "eks_observability_accelerator" { aws_region = var.aws_region eks_cluster_id = var.eks_cluster_id - # TODO: create also a cluster, VPC -- check if enough VPCs - # deploys AWS Distro for OpenTelemetry operator into the cluster enable_amazon_eks_adot = true @@ -16,7 +14,7 @@ module "eks_observability_accelerator" { enable_cert_manager = true # # -- or enable opentelemetry operator - enable_opentelemetry_operator = false #-- true doesn't work for me, needs fix + enable_opentelemetry_operator = false #open_telemetry_operator_config = map() // custom config # creates a new AMP workspace, defaults to true @@ -34,11 +32,9 @@ module "eks_observability_accelerator" { managed_grafana_workspace_id = var.managed_grafana_workspace_id grafana_api_key = var.grafana_api_key - - - enable_java = true - enable_java_recording_rules = true - + # enable workload-specific collector, metrics, alerts and dashboards + enable_java = false + enable_java_recording_rules = false # enable_haproxy = true # haproxy_config = { @@ -46,29 +42,10 @@ module "eks_observability_accelerator" { # grafana_endpoint = module.grafana.endpoint # } - - # java_config = { - # amp_endpoint = "" - # grafana_endpoint = "" - # } - - # # -- or provide custom alerts definition - # prometheus_custom_alert_rule = var.prometheus_custom_alert_rule - - - # # create grafana workspace, and customer to deal with authentication later - # create_managed_grafana_workspace = true - # grafana_auth_provider = var.grafana_auth_provider //SAML or AWS_SSO - # grafana_account_access_type = var.grafana_account_access_type // CURRENT_ACCOUNT or ORGANIZATION - # grafana_permission_type = var.grafana_permission_type // SERVICE_MANAGED or CUSTOMER_MANAGED - # grafana_permission_role_arn = var.grafana_permission_role_arn // if CUSTOMER_MANAGED - - # # -- or using existing amg workspace. so we can use API for keys - + enable_infra_metrics = true + #infra_metrics_config = {} tags = local.tags - - } # module "amp" { diff --git a/main.tf b/main.tf index ac8af93..1a84cff 100644 --- a/main.tf +++ b/main.tf @@ -81,3 +81,41 @@ module "java" { module.operator ] } + +module "infra" { + count = var.enable_infra_metrics ? 1 : 0 + source = "./modules/workloads/kube-prometheus" + + addon_context = local.context + + amp_endpoint = local.amp_ws_endpoint + amp_id = local.amp_ws_id + amp_region = local.amp_ws_region + + config = var.infra_metrics_config + + + # enable_kube_state_metrics = var.infra_metrics_config.enable_kube_state_metrics + # kms_create_namespace = var.infra_metrics_config.kms_create_namespace + # ksm_k8s_namespace = var.infra_metrics_config.ksm_k8s_namespace + # ksm_helm_chart_name = var.infra_metrics_config.ksm_helm_chart_name + # ksm_helm_chart_version = var.infra_metrics_config.ksm_helm_chart_version + # ksm_helm_release_name = var.infra_metrics_config.ksm_helm_release_name + # ksm_helm_repo_url = var.infra_metrics_config.ksm_helm_repo_url + # ksm_helm_settings = var.infra_metrics_config.ksm_helm_settings + # ksm_helm_values = var.infra_metrics_config.ksm_helm_values + + # enable_node_exporter = var.infra_metrics_config.enable_node_exporter + # ne_create_namespace = var.infra_metrics_config.ne_create_namespace + # ne_helm_chart_name = var.infra_metrics_config.ne_helm_chart_name + # ne_helm_chart_version = var.infra_metrics_config.ne_helm_chart_version + # ne_helm_release_name = var.infra_metrics_config.ne_helm_release_name + # ne_helm_repo_url = var.infra_metrics_config.ne_helm_repo_url + # ne_helm_settings = var.infra_metrics_config.ksm_helm_settings + # ne_helm_values = var.infra_metrics_config.ksm_helm_values + + depends_on = [ + module.operator + ] + +} diff --git a/modules/workloads/kube-prometheus/README.md b/modules/workloads/kube-prometheus/README.md new file mode 100644 index 0000000..16df589 --- /dev/null +++ b/modules/workloads/kube-prometheus/README.md @@ -0,0 +1,49 @@ +# Observability Pattern for Java/JMX + +This module provides an automated experience around Observability for Nginx workloads. +It provides the following resources: + +- AWS Distro For OpenTelemetry Operator and Collector +- AWS Managed Grafana Dashboard and data source +- Alerts and recording rules with AWS Managed Service for Prometheus + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [aws](#requirement\_aws) | >= 3.72 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.72 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [helm\_addon](#module\_helm\_addon) | ../helm-addon | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [addon\_context](#input\_addon\_context) | Input configuration for the addon | 
object({
    aws_caller_identity_account_id = string
    aws_caller_identity_arn        = string
    aws_eks_cluster_endpoint       = string
    aws_partition_id               = string
    aws_region_name                = string
    eks_cluster_id                 = string
    eks_oidc_issuer_url            = string
    eks_oidc_provider_arn          = string
    irsa_iam_permissions_boundary  = string
    irsa_iam_role_path             = string
    tags                           = map(string)
  })
| n/a | yes | +| [amazon\_prometheus\_workspace\_endpoint](#input\_amazon\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `null` | no | +| [amazon\_prometheus\_workspace\_region](#input\_amazon\_prometheus\_workspace\_region) | Amazon Managed Prometheus Workspace's Region | `string` | `null` | no | +| [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no | + +## Outputs + +No outputs. + diff --git a/modules/workloads/kube-prometheus/main.tf b/modules/workloads/kube-prometheus/main.tf new file mode 100644 index 0000000..5f9a511 --- /dev/null +++ b/modules/workloads/kube-prometheus/main.tf @@ -0,0 +1,104 @@ +locals { + name = "adot-collector-kubeprometheus" + namespace = try(var.config.helm_config.namespace, local.name) +} +resource "helm_release" "kube_state_metrics" { + count = var.config.enable_kube_state_metrics ? 1 : 0 + chart = var.config.ksm_helm_chart_name + create_namespace = var.config.kms_create_namespace + namespace = var.config.ksm_k8s_namespace + name = var.config.ksm_helm_release_name + version = var.config.ksm_helm_chart_version + repository = var.config.ksm_helm_repo_url + + # values = [ + # var.config.ksm_helm_values + # ] + + dynamic "set" { + for_each = var.config.ksm_helm_settings + content { + name = set.key + value = set.value + } + } +} + +resource "helm_release" "prometheus_node_exporter" { + count = var.config.enable_node_exporter ? 1 : 0 + chart = var.config.ne_helm_chart_name + create_namespace = var.config.ne_create_namespace + namespace = var.config.ne_k8s_namespace + name = var.config.ne_helm_release_name + version = var.config.ne_helm_chart_version + repository = var.config.ne_helm_repo_url + + # values = [ + # var.config.ne_helm_values + # ] + + dynamic "set" { + for_each = var.config.ne_helm_settings + content { + name = set.key + value = set.value + } + } +} +data "aws_partition" "current" {} + +module "helm_addon" { + source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon" + + helm_config = merge( + { + name = local.name + chart = "${path.module}/otel-config" + version = "0.2.0" + namespace = local.namespace + description = "ADOT helm Chart deployment configuration" + }, + var.helm_config + ) + + set_values = [ + { + name = "ampurl" + value = "${var.amp_endpoint}api/v1/remote_write" + }, + { + name = "region" + value = var.amp_region + }, + { + name = "prometheusMetricsEndpoint" + value = "metrics" + }, + { + name = "prometheusMetricsPort" + value = 8888 + }, + { + name = "scrapeInterval" + value = "15s" + }, + { + name = "scrapeTimeout" + value = "10s" + }, + { + name = "scrapeSampleLimit" + value = 1000 + } + ] + + irsa_config = { + create_kubernetes_namespace = true + kubernetes_namespace = local.namespace + create_kubernetes_service_account = true + kubernetes_service_account = try(var.config.helm_config.service_account, local.name) + irsa_iam_policies = ["arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess"] + } + + addon_context = var.addon_context +} diff --git a/modules/workloads/kube-prometheus/otel-config/Chart.yaml b/modules/workloads/kube-prometheus/otel-config/Chart.yaml new file mode 100644 index 0000000..94c1fbd --- /dev/null +++ b/modules/workloads/kube-prometheus/otel-config/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: opentelemetry +description: A Helm chart to install otel operator +type: application +version: 0.2.0 +appVersion: v0.1.0 diff --git a/modules/workloads/kube-prometheus/otel-config/templates/clusterrole.yaml b/modules/workloads/kube-prometheus/otel-config/templates/clusterrole.yaml new file mode 100644 index 0000000..4bb1fb7 --- /dev/null +++ b/modules/workloads/kube-prometheus/otel-config/templates/clusterrole.yaml @@ -0,0 +1,29 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-prometheus-role +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch + - nonResourceURLs: + - /metrics + verbs: + - get diff --git a/modules/workloads/kube-prometheus/otel-config/templates/clusterrolebinding.yaml b/modules/workloads/kube-prometheus/otel-config/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..992d769 --- /dev/null +++ b/modules/workloads/kube-prometheus/otel-config/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-prometheus-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-prometheus-role +subjects: + - kind: ServiceAccount + name: adot-collector-kubeprometheus + namespace: adot-collector-kubeprometheus diff --git a/modules/workloads/kube-prometheus/otel-config/templates/opentelemetrycollector.yaml b/modules/workloads/kube-prometheus/otel-config/templates/opentelemetrycollector.yaml new file mode 100644 index 0000000..a18b39a --- /dev/null +++ b/modules/workloads/kube-prometheus/otel-config/templates/opentelemetrycollector.yaml @@ -0,0 +1,1593 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: adot +spec: + image: public.ecr.aws/aws-observability/aws-otel-collector:latest + mode: deployment + serviceAccount: adot-collector-kubeprometheus + config: | + receivers: + prometheus: + config: + global: + scrape_interval: {{ .Values.scrapeInterval }} + scrape_timeout: {{ .Values.scrapeTimeout }} + scrape_configs: + - job_name: 'kubernetes-kubelet' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc.cluster.local:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics + - job_name: 'kubelet' + metrics_path: /metrics/cadvisor + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc.cluster.local:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (prometheus-node-exporter);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-prometheus);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] + separator: ; + regex: (true);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-web + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-web + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-operator/0 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + tls_config: + ca_file: /etc/prometheus/certs/secret_default_kube-prometheus-stack-admission_ca + server_name: kube-prometheus-stack-operator + insecure_skip_verify: false + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-operator);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/2 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics/probes + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/1 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics/cadvisor + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/0 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-state-metrics/0 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_instance, __meta_kubernetes_service_labelpresent_app_kubernetes_io_instance] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kube-state-metrics);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-scheduler/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-scheduler);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-proxy/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-proxy);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-etcd/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-etcd);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-controller-manager/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-controller-manager);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-coredns/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-coredns);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + namespaces: + own_namespace: false + names: + - kube-system + + - job_name: serviceMonitor/default/kube-prometheus-stack-apiserver/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server_name: kubernetes + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_component, __meta_kubernetes_service_labelpresent_component] + separator: ; + regex: (kubernetes);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_component] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-alertmanager/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-alertmanager);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] + separator: ; + regex: (true);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-web + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-web + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + + exporters: + awsprometheusremotewrite: + endpoint: {{ .Values.ampurl }} + aws_auth: + region: {{ .Values.region }} + service: "aps" + logging: + loglevel: info + extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + service: + extensions: [pprof, zpages, health_check] + pipelines: + metrics: + receivers: [prometheus] + exporters: [logging, awsprometheusremotewrite] diff --git a/modules/workloads/kube-prometheus/otel-config/values.yaml b/modules/workloads/kube-prometheus/otel-config/values.yaml new file mode 100644 index 0000000..f3517b2 --- /dev/null +++ b/modules/workloads/kube-prometheus/otel-config/values.yaml @@ -0,0 +1,7 @@ +ampurl: ${amp_url} +region: ${region} +prometheusMetricsEndpoint: ${prometheus_metrics_endpoint} +prometheusMetricsPort: ${prometheus_metrics_port} +scrapeInterval: ${scrape_interval} +scrapeTimeout: ${scrape_timeout} +scrapeSampleLimit: ${scrape_sample_limit} diff --git a/modules/workloads/kube-prometheus/outputs.tf b/modules/workloads/kube-prometheus/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/workloads/kube-prometheus/variables.tf b/modules/workloads/kube-prometheus/variables.tf new file mode 100644 index 0000000..b930cb3 --- /dev/null +++ b/modules/workloads/kube-prometheus/variables.tf @@ -0,0 +1,198 @@ + + +# ADOT variable +variable "helm_config" { + description = "Helm Config for Prometheus" + type = any + default = {} +} + +variable "amp_endpoint" { + description = "Amazon Managed Prometheus Workspace Endpoint" + type = string + default = null +} +variable "amp_id" { + description = "Amazon Managed Prometheus Workspace ID" + type = string + default = null +} + +variable "amp_region" { + description = "Amazon Managed Prometheus Workspace's Region" + type = string + default = null +} + +variable "addon_context" { + description = "Input configuration for the addon" + type = object({ + aws_caller_identity_account_id = string + aws_caller_identity_arn = string + aws_eks_cluster_endpoint = string + aws_partition_id = string + aws_region_name = string + eks_cluster_id = string + eks_oidc_issuer_url = string + eks_oidc_provider_arn = string + irsa_iam_permissions_boundary = string + irsa_iam_role_path = string + tags = map(string) + }) +} + +variable "config" { + type = object({ + helm_config = map(any) + + enable_kube_state_metrics = bool + kms_create_namespace = bool + ksm_k8s_namespace = string + ksm_helm_chart_name = string + ksm_helm_chart_version = string + ksm_helm_release_name = string + ksm_helm_repo_url = string + ksm_helm_settings = map(string) + ksm_helm_values = map(any) + + enable_node_exporter = bool + ne_create_namespace = bool + ne_k8s_namespace = string + ne_helm_chart_name = string + ne_helm_chart_version = string + ne_helm_release_name = string + ne_helm_repo_url = string + ne_helm_settings = map(string) + ne_helm_values = map(any) + }) + + default = { + enable_kube_state_metrics = true + enable_node_exporter = true + helm_config = {} + + kms_create_namespace = true + ksm_helm_chart_name = "kube-state-metrics" + ksm_helm_chart_version = "4.9.2" + ksm_helm_release_name = "kube-state-metrics" + ksm_helm_repo_url = "https://prometheus-community.github.io/helm-charts" + ksm_helm_settings = {} + ksm_helm_values = {} + ksm_k8s_namespace = "kube-system" + + ne_create_namespace = true + ne_k8s_namespace = "prometheus-node-exporter" + ne_helm_chart_name = "prometheus-node-exporter" + ne_helm_chart_version = "2.0.3" + ne_helm_release_name = "prometheus-node-exporter" + ne_helm_repo_url = "https://prometheus-community.github.io/helm-charts" + ne_helm_settings = {} + ne_helm_values = {} + } + nullable = false +} + +# # Kube-state-metrics Variables +# variable "enable_kube_state_metrics" { +# description = "Variable indicating whether deployment is enabled" +# type = bool +# default = true +# } + +# # Helm + +# variable "helm_create_namespace_ksm" { +# type = bool +# default = true +# description = "Create the namespace if it does not yet exist" +# } + +# variable "helm_chart_name_ksm" { +# type = string +# default = "kube-state-metrics" +# description = "Helm chart name to be installed" +# } + +# variable "helm_chart_version_ksm" { +# type = string +# default = "4.9.2" +# description = "Version of the Helm chart" +# } + +# variable "helm_release_name_ksm" { +# type = string +# default = "kube-state-metrics" +# description = "Helm release name" +# } + +# variable "helm_repo_url_ksm" { +# type = string +# default = "https://prometheus-community.github.io/helm-charts" +# description = "Helm repository" +# } + +# # K8s +# variable "k8s_namespace_ksm" { +# type = string +# default = "kube-system" +# description = "The K8s namespace in which the prometheus-node-exporter service account has been created" +# } + +# variable "settings" { +# type = map(any) +# default = {} +# description = "Additional settings which will be passed to the Helm chart values, see https://hub.helm.sh/charts/stable/prometheus-node-exporter" +# } + +# variable "values" { +# type = string +# default = "" +# description = "Additional yaml encoded values which will be passed to the Helm chart." +# } + +# # Node exporter Variables + +# variable "enabled_node_exporter" { +# description = "Variable indicating whether deployment is enabled" +# type = bool +# default = true +# } + +# # Helm + +# variable "helm_create_namespace_ne" { +# type = bool +# default = true +# description = "Create the namespace if it does not yet exist" +# } + +# variable "helm_chart_name_ne" { +# type = string +# default = "prometheus-node-exporter" +# description = "Helm chart name to be installed" +# } + +# variable "helm_chart_version_ne" { +# type = string +# default = "2.0.3" +# description = "Version of the Helm chart" +# } + +# variable "helm_release_name_ne" { +# type = string +# default = "prometheus-node-exporter" +# description = "Helm release name" +# } + +# variable "helm_repo_url_ne" { +# type = string +# default = "https://prometheus-community.github.io/helm-charts" +# description = "Helm repository" +# } + +# # K8s +# variable "helm_repo_url_ne" { +# type = string +# default = "kube-system" +# description = "The K8s namespace in which the prometheus-node-exporter service account has been created" +# } diff --git a/modules/workloads/kube-prometheus/versions.tf b/modules/workloads/kube-prometheus/versions.tf new file mode 100644 index 0000000..d2ddf87 --- /dev/null +++ b/modules/workloads/kube-prometheus/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + } +} diff --git a/variables.tf b/variables.tf index 0f26a1e..74cb9b3 100644 --- a/variables.tf +++ b/variables.tf @@ -111,3 +111,61 @@ variable "tags" { type = map(string) default = {} } + + +variable "enable_infra_metrics" { + type = bool + default = false +} + +variable "infra_metrics_config" { + type = object({ + helm_config = map(any) + + enable_kube_state_metrics = bool + kms_create_namespace = bool + ksm_k8s_namespace = string + ksm_helm_chart_name = string + ksm_helm_chart_version = string + ksm_helm_release_name = string + ksm_helm_repo_url = string + ksm_helm_settings = map(string) + ksm_helm_values = map(any) + + enable_node_exporter = bool + ne_create_namespace = bool + ne_k8s_namespace = string + ne_helm_chart_name = string + ne_helm_chart_version = string + ne_helm_release_name = string + ne_helm_repo_url = string + ne_helm_settings = map(string) + ne_helm_values = map(any) + }) + + default = null + + # default = { + # enable_kube_state_metrics = true + # enable_node_exporter = true + # helm_config = {} + + # kms_create_namespace = true + # ksm_helm_chart_name = "kube-state-metrics" + # ksm_helm_chart_version = "4.9.2" + # ksm_helm_release_name = "kube-state-metrics" + # ksm_helm_repo_url = "https://prometheus-community.github.io/helm-charts" + # ksm_helm_settings = {} + # ksm_helm_values = {} + # ksm_k8s_namespace = "kube-system" + + # ne_create_namespace = true + # ne_k8s_namespace = "prometheus-node-exporter" + # ne_helm_chart_name = "prometheus-node-exporter" + # ne_helm_chart_version = "2.0.3" + # ne_helm_release_name = "prometheus-node-exporter" + # ne_helm_repo_url = "https://prometheus-community.github.io/helm-charts" + # ne_helm_settings = {} + # ne_helm_values = {} + # } +}