diff --git a/docs/eks/index.md b/docs/eks/index.md index c1d98d9..b6c3ef8 100644 --- a/docs/eks/index.md +++ b/docs/eks/index.md @@ -167,3 +167,18 @@ sum(up{job="custom-metrics"}) by (container_name, cluster, nodename) ``` Screenshot 2023-01-31 at 11 16 21 + +## Troubleshooting + +When you upgrade the eks-monitoring module from v2.1.0 or earlier, the following error may occur. + +```bash +Error: cannot patch "prometheus-node-exporter" with kind DaemonSet: DaemonSet.apps "prometheus-node-exporter" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/instance":"prometheus-node-exporter", "app.kubernetes.io/name":"prometheus-node-exporter"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +This is due to the upgrade of the node-exporter chart from v2 to v4. Manually delete the node-exporter's DaemonSet as described in [the link here](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter#3x-to-4x), and then apply. + +```bash +kubectl -n prometheus-node-exporter delete daemonset -l app=prometheus-node-exporter +terraform apply +``` diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 9615f9c..6300df3 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -85,7 +85,7 @@ This module makes use of the open source [kube-prometheus-stack](https://github. | [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | | [managed\_prometheus\_workspace\_region](#input\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus Workspace's Region | `string` | `null` | no | -| [ne\_config](#input\_ne\_config) | Node exporter configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "prometheus-node-exporter",
  "helm_chart_version": "2.0.3",
  "helm_release_name": "prometheus-node-exporter",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "prometheus-node-exporter",
  "scrape_interval": "60s",
  "scrape_timeout": "60s"
}
| no | +| [ne\_config](#input\_ne\_config) | Node exporter configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "prometheus-node-exporter",
  "helm_chart_version": "4.14.0",
  "helm_release_name": "prometheus-node-exporter",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "prometheus-node-exporter",
  "scrape_interval": "60s",
  "scrape_timeout": "60s"
}
| no | | [nginx\_config](#input\_nginx\_config) | Configuration object for NGINX monitoring | 
object({
    enable_alerting_rules       = bool
    scrape_sample_limit         = number
    prometheus_metrics_endpoint = string
  })
| 
{
  "enable_alerting_rules": true,
  "prometheus_metrics_endpoint": "metrics",
  "scrape_sample_limit": 1000
}
| no | | [prometheus\_config](#input\_prometheus\_config) | Controls default values such as scrape interval, timeouts and ports globally | 
object({
    global_scrape_interval = string
    global_scrape_timeout  = string
  })
| 
{
  "global_scrape_interval": "60s",
  "global_scrape_timeout": "15s"
}
| no | | [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no | @@ -99,3 +99,18 @@ This module makes use of the open source [kube-prometheus-stack](https://github. | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | | [grafana\_dashboard\_urls](#output\_grafana\_dashboard\_urls) | URLs for dashboards created | + +## Troubleshooting + +When you upgrade the eks-monitoring module from v2.1.0 or earlier, the following error may occur. + +```bash +Error: cannot patch "prometheus-node-exporter" with kind DaemonSet: DaemonSet.apps "prometheus-node-exporter" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/instance":"prometheus-node-exporter", "app.kubernetes.io/name":"prometheus-node-exporter"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +This is due to the upgrade of the node-exporter chart from v2 to v4. Manually delete the node-exporter's DaemonSet as described in [the link here](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter#3x-to-4x), and then apply. + +```bash +kubectl -n prometheus-node-exporter delete daemonset -l app=prometheus-node-exporter +terraform apply +``` diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index 383f6f5..828c0e2 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -131,7 +131,7 @@ variable "ne_config" { default = { create_namespace = true helm_chart_name = "prometheus-node-exporter" - helm_chart_version = "2.0.3" + helm_chart_version = "4.14.0" helm_release_name = "prometheus-node-exporter" helm_repo_url = "https://prometheus-community.github.io/helm-charts" helm_settings = {}