From 86cc9489d2970efdd833bc948c674972823641a9 Mon Sep 17 00:00:00 2001 From: Jens-Uwe Walther Date: Fri, 25 Aug 2023 10:36:35 +0200 Subject: [PATCH] fix(eks-monitoring): kube-proxy scrape config and alert rule (#221) * changes to just OTEL kube-proxy scrape job * agree with koffir@ to implement kube-proxy by default * re-added KubeProxyDown Prometheus alert rule * added kube-proxy URL related stuff * corrected kube-proxy stuff according to new structure * Use SEMVER for flux now * Run pre-commit * Bump OTEL chart version * Substitute URL path with SEMVER * Run pre-commit * Added kube-proxy config object * Run pre-commit * Use kube-proxy config object for dashboards --------- Co-authored-by: Jens-Uwe Walther --- modules/eks-monitoring/README.md | 19 +-- modules/eks-monitoring/alerts.tf | 8 ++ modules/eks-monitoring/dashboards.tf | 22 ++++ modules/eks-monitoring/locals.tf | 43 +++--- modules/eks-monitoring/otel-config/Chart.yaml | 4 +- .../templates/opentelemetrycollector.yaml | 123 +++--------------- modules/eks-monitoring/variables.tf | 40 ++++-- 7 files changed, 121 insertions(+), 138 deletions(-) diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 0033b23..34deaf7 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -59,6 +59,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [kubectl_manifest.api_server_dashboards](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.flux_gitrepository](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.flux_kustomization](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.kubeproxy_monitoring_dashboard](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -69,7 +70,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [adot\_loglevel](#input\_adot\_loglevel) | Verbosity level for ADOT collector logs. This accepts (detailed\|normal\|basic), see https://aws-otel.github.io/docs/components/misc-exporters for mor infos. | `string` | `"normal"` | no | -| [adothealth\_monitoring\_config](#input\_adothealth\_monitoring\_config) | Config object for API server monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      grafana_adothealth_dashboard_url = string
    })
  })
| `null` | no | +| [adothealth\_monitoring\_config](#input\_adothealth\_monitoring\_config) | Config object for ADOT health monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      grafana_adothealth_dashboard_url = string
    })
  })
| `null` | no | | [apiserver\_monitoring\_config](#input\_apiserver\_monitoring\_config) | Config object for API server monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      basic           = string
      advanced        = string
      troubleshooting = string
    })
  })
| `null` | no | | [custom\_metrics\_config](#input\_custom\_metrics\_config) | Configuration object to enable custom metrics collection | 
map(object({
    enableBasicAuth       = bool
    path                  = string
    basicAuthUsername     = string
    basicAuthPassword     = string
    ports                 = string
    droppedSeriesPrefixes = string
  }))
| `null` | no | | [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes | @@ -92,26 +93,28 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [enable\_recording\_rules](#input\_enable\_recording\_rules) | Enables or disables Managed Prometheus recording rules | `bool` | `true` | no | | [enable\_tracing](#input\_enable\_tracing) | Enables tracing with OTLP traces receiver to X-Ray | `bool` | `true` | no | | [flux\_config](#input\_flux\_config) | FluxCD configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "flux2",
  "helm_chart_version": "2.7.0",
  "helm_release_name": "observability-fluxcd-addon",
  "helm_repo_url": "https://fluxcd-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "flux-system"
}
| no | -| [flux\_gitrepository\_branch](#input\_flux\_gitrepository\_branch) | Flux GitRepository Branch | `string` | `"main"` | no | +| [flux\_gitrepository\_branch](#input\_flux\_gitrepository\_branch) | Flux GitRepository Branch | `string` | `"v0.2.0"` | no | | [flux\_gitrepository\_name](#input\_flux\_gitrepository\_name) | Flux GitRepository name | `string` | `"aws-observability-accelerator"` | no | | [flux\_gitrepository\_url](#input\_flux\_gitrepository\_url) | Flux GitRepository URL | `string` | `"https://github.com/aws-observability/aws-observability-accelerator"` | no | | [flux\_kustomization\_name](#input\_flux\_kustomization\_name) | Flux Kustomization name | `string` | `"grafana-dashboards-infrastructure"` | no | | [flux\_kustomization\_path](#input\_flux\_kustomization\_path) | Flux Kustomization Path | `string` | `"./artifacts/grafana-operator-manifests/eks/infrastructure"` | no | | [go\_config](#input\_go\_config) | Grafana Operator configuration | 
object({
    create_namespace   = bool
    helm_chart         = string
    helm_name          = string
    k8s_namespace      = string
    helm_release_name  = string
    helm_chart_version = string
  })
| 
{
  "create_namespace": true,
  "helm_chart": "oci://ghcr.io/grafana-operator/helm-charts/grafana-operator",
  "helm_chart_version": "v5.0.0-rc3",
  "helm_name": "grafana-operator",
  "helm_release_name": "grafana-operator",
  "k8s_namespace": "grafana-operator"
}
| no | | [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace. Required if `enable_external_secrets = true` | `string` | `""` | no | -| [grafana\_cluster\_dashboard\_url](#input\_grafana\_cluster\_dashboard\_url) | Dashboard URL for Cluster Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/cluster.json"` | no | -| [grafana\_kubelet\_dashboard\_url](#input\_grafana\_kubelet\_dashboard\_url) | Dashboard URL for Kubelet Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json"` | no | -| [grafana\_namespace\_workloads\_dashboard\_url](#input\_grafana\_namespace\_workloads\_dashboard\_url) | Dashboard URL for Namespace Workloads Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json"` | no | -| [grafana\_node\_exporter\_dashboard\_url](#input\_grafana\_node\_exporter\_dashboard\_url) | Dashboard URL for Node Exporter Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json"` | no | -| [grafana\_nodes\_dashboard\_url](#input\_grafana\_nodes\_dashboard\_url) | Dashboard URL for Nodes Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodes.json"` | no | +| [grafana\_cluster\_dashboard\_url](#input\_grafana\_cluster\_dashboard\_url) | Dashboard URL for Cluster Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/cluster.json"` | no | +| [grafana\_kubelet\_dashboard\_url](#input\_grafana\_kubelet\_dashboard\_url) | Dashboard URL for Kubelet Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json"` | no | +| [grafana\_kubeproxy\_dashboard\_url](#input\_grafana\_kubeproxy\_dashboard\_url) | Dashboard URL for kube-proxy Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/kube-proxy/kube-proxy.json"` | no | +| [grafana\_namespace\_workloads\_dashboard\_url](#input\_grafana\_namespace\_workloads\_dashboard\_url) | Dashboard URL for Namespace Workloads Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json"` | no | +| [grafana\_node\_exporter\_dashboard\_url](#input\_grafana\_node\_exporter\_dashboard\_url) | Dashboard URL for Node Exporter Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json"` | no | +| [grafana\_nodes\_dashboard\_url](#input\_grafana\_nodes\_dashboard\_url) | Dashboard URL for Nodes Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/nodes.json"` | no | | [grafana\_url](#input\_grafana\_url) | Endpoint URL of Amazon Managed Grafana workspace. Required if `enable_grafana_operator = true` | `string` | `""` | no | -| [grafana\_workloads\_dashboard\_url](#input\_grafana\_workloads\_dashboard\_url) | Dashboard URL for Workloads Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/workloads.json"` | no | +| [grafana\_workloads\_dashboard\_url](#input\_grafana\_workloads\_dashboard\_url) | Dashboard URL for Workloads Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/workloads.json"` | no | | [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no | | [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `null` | no | | [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no | | [istio\_config](#input\_istio\_config) | Configuration object for ISTIO monitoring | 
object({
    enable_alerting_rules  = bool
    enable_recording_rules = bool
    enable_dashboards      = bool
    scrape_sample_limit    = number

    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    managed_prometheus_workspace_id = string
    prometheus_metrics_endpoint     = string

    dashboards = object({
      cp          = string
      mesh        = string
      performance = string
      service     = string
    })
  })
| `null` | no | | [java\_config](#input\_java\_config) | Configuration object for Java/JMX monitoring | 
object({
    enable_alerting_rules  = bool
    enable_recording_rules = bool
    enable_dashboards      = bool
    scrape_sample_limit    = number


    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    grafana_dashboard_url = string

    prometheus_metrics_endpoint = string
  })
| `null` | no | | [ksm\_config](#input\_ksm\_config) | Kube State metrics configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "kube-state-metrics",
  "helm_chart_version": "4.24.0",
  "helm_release_name": "kube-state-metrics",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "kube-system",
  "scrape_interval": "60s",
  "scrape_timeout": "15s"
}
| no | +| [kubeproxy\_monitoring\_config](#input\_kubeproxy\_monitoring\_config) | Config object for kube-proxy monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      grafana_kubeproxy_dashboard_url = string
    })
  })
| `null` | no | | [logs\_config](#input\_logs\_config) | Configuration object for logs collection | 
object({
    cw_log_retention_days = number
  })
| 
{
  "cw_log_retention_days": 90
}
| no | | [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | diff --git a/modules/eks-monitoring/alerts.tf b/modules/eks-monitoring/alerts.tf index 6d4d591..1cae59b 100644 --- a/modules/eks-monitoring/alerts.tf +++ b/modules/eks-monitoring/alerts.tf @@ -262,6 +262,14 @@ groups: annotations: description: Kubelet has disappeared from Prometheus target discovery. summary: Target disappeared from Prometheus target discovery. + - alert: KubeProxyDown + expr: absent(up{job="kube-proxy"} == 1) + for: 15m + labels: + severity: critical + annotations: + description: KubeProxy has disappeared from Prometheus target discovery. + summary: Target disappeared from Prometheus target discovery. - alert: KubeVersionMismatch expr: count by(cluster) (count by(git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"}, "git_version", "$1", "git_version", "(v[0-9]*.[0-9]*).*"))) > 1 for: 15m diff --git a/modules/eks-monitoring/dashboards.tf b/modules/eks-monitoring/dashboards.tf index 0e5785e..a73a557 100644 --- a/modules/eks-monitoring/dashboards.tf +++ b/modules/eks-monitoring/dashboards.tf @@ -94,3 +94,25 @@ YAML count = var.enable_adotcollector_metrics ? 1 : 0 depends_on = [module.external_secrets] } + +resource "kubectl_manifest" "kubeproxy_monitoring_dashboard" { + yaml_body = <