Merge alerts and rules in two separate rules

2026-10-09 00:09:43 +00:00 · 2022-08-29 18:55:55 +02:00
parent eadacc98c2
commit c3e55fb262
3 changed files with 4 additions and 247 deletions
@@ -2,7 +2,7 @@
 # Alerting rules ###############################################################################################################################
 ################################################################################################################################################

-resource "aws_prometheus_rule_group_namespace" "nodenw" {
+resource "aws_prometheus_rule_group_namespace" "alerting_rules" {
  count = var.enable_alerting_rules ? 1 : 0

  name         = "nodenw-rules"
@@ -19,18 +19,6 @@ groups:
        annotations:
          description: Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}
          summary: Network interface is often changing its status
-EOF
-}
-
-
-
-resource "aws_prometheus_rule_group_namespace" "nodeexporter" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "nodeexporter-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: nodeexp-01
    rules:
      - alert: NodeFilesystemSpaceFillingUp
@@ -208,16 +196,6 @@ groups:
        annotations:
          description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
          summary: Kernel is predicted to exhaust file descriptors limit soon.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyschdlr" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyschdlr-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubesysschdlr-01
    rules:
      - alert: KubeSchedulerDown
@@ -228,16 +206,6 @@ groups:
        annotations:
          description: KubeScheduler has disappeared from Prometheus target discovery.
          summary: Target disappeared from Prometheus target discovery.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyskblt" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyskblt-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubesyskblt-01
    rules:
      - alert: KubeNodeNotReady
@@ -364,17 +332,6 @@ groups:
        annotations:
          description: Kubelet has disappeared from Prometheus target discovery.
          summary: Target disappeared from Prometheus target discovery.
-
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyskbpxy" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyskbpxy-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubesyspxy-01
    rules:
      - alert: KubeProxyDown
@@ -385,16 +342,6 @@ groups:
        annotations:
          description: KubeProxy has disappeared from Prometheus target discovery.
          summary: Target disappeared from Prometheus target discovery.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesys" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesys-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubesys-01
    rules:
      - alert: KubeVersionMismatch
@@ -415,17 +362,6 @@ groups:
        annotations:
          description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
          summary: Kubernetes API server client is experiencing errors.
-
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyscm" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyscm-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubesyscm-01
    rules:
      - alert: KubeControllerManagerDown
@@ -436,16 +372,6 @@ groups:
        annotations:
          description: KubeControllerManager has disappeared from Prometheus target discovery.
          summary: Target disappeared from Prometheus target discovery.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesysapi" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesysapi-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubesysapi-01
    rules:
      - alert: KubeClientCertificateExpiration
@@ -503,18 +429,6 @@ groups:
        annotations:
          description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
          summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
-EOF
-}
-
-# Default limit of 10 needs to be raised
-
-resource "aws_prometheus_rule_group_namespace" "kubestorage" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubestorage-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubestg-01
    rules:
      - alert: KubePersistentVolumeFillingUp
@@ -565,16 +479,6 @@ groups:
        annotations:
          description: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
          summary: PersistentVolume is having issues with provisioning.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kuberesources" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kuberesources-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kuberes-01
    rules:
      - alert: KubeCPUOvercommit
@@ -655,16 +559,6 @@ groups:
        annotations:
          description: The {{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.
          summary: Processes experience elevated CPU throttling.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubeapps" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubeapps-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubeapps-01
    rules:
      - alert: KubePodCrashLooping
@@ -814,18 +708,6 @@ groups:
        annotations:
          description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.
          summary: HPA is running at max replicas
-EOF
-}
-
-
-
-resource "aws_prometheus_rule_group_namespace" "kubestm" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubestm-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubestm-01
    rules:
      - alert: KubeStateMetricsListErrors
@@ -866,16 +748,6 @@ groups:
        annotations:
          description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed.
          summary: kube-state-metrics shards are missing.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "apislos" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "api-slos"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: apislos-01
    rules:
      - alert: KubeAPIErrorBudgetBurn
@@ -924,17 +796,6 @@ groups:
        annotations:
          description: The API server is burning too much error budget.
          summary: The API server is burning too much error budget.
-EOF
-}
-
-
-resource "aws_prometheus_rule_group_namespace" "generic" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "generic-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: general-01
    rules:
      - alert: TargetDown
@@ -960,16 +821,6 @@ groups:
          severity: none
        annotations:
          description: This is an alert that is used to inhibit info alerts. By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with other alerts. This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a severity of 'warning' or 'critical' starts firing on the same namespace. This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "etcd" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "etcd-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: etcd-01
    rules:
      - alert: etcdInsufficientMembers
@@ -44,7 +44,7 @@ resource "helm_release" "prometheus_node_exporter" {
 }

 module "helm_addon" {
-  source = "github.com/aws-observability/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
+  source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"

  helm_config = merge(
    {
@@ -4,10 +4,9 @@
 # Recording rules ##############################################################################################################################
 ################################################################################################################################################

-resource "aws_prometheus_rule_group_namespace" "noderules" {
+resource "aws_prometheus_rule_group_namespace" "recording_rules" {
  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "noderules-recording"
+  name         = "infra-recording-rules"
  workspace_id = var.managed_prometheus_workspace_id
  data         = <<EOF
 groups:
@@ -27,16 +26,6 @@ groups:
    rules:
      - record: cluster:node_cpu:ratio_rate5m
        expr: sum(rate(node_cpu_seconds_total{job="node-exporter",mode!="idle",mode!="iowait",mode!="steal"}[5m])) / count(sum by(cluster, instance, cpu) (node_cpu_seconds_total{job="node-exporter"}))
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubelet" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubelet-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubelet-01
    rules:
      - record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
@@ -55,17 +44,6 @@ groups:
        expr: histogram_quantile(0.5, sum by(cluster, instance, le) (rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        labels:
          quantile: 0.5
-
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "ne" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "ne-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: nerules-01
    rules:
      - record: instance:node_num_cpu:sum
@@ -110,16 +88,6 @@ groups:
    rules:
      - record: instance:node_network_transmit_drop_excluding_lo:rate5m
        expr: sum without(device) (rate(node_network_transmit_drop_total{device!="lo",job="node-exporter"}[5m]))
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubescheduler" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubescheduler-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubeschdlr-01
    rules:
      - record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
@@ -174,16 +142,6 @@ groups:
        expr: histogram_quantile(0.5, sum without(instance, pod) (rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])))
        labels:
          quantile: 0.5
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubepromnr" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubeprom-noderecording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubepromnr-01
    rules:
      - record: instance:node_cpu:rate:sum
@@ -208,16 +166,6 @@ groups:
    rules:
      - record: cluster:node_cpu:ratio
        expr: cluster:node_cpu:sum_rate5m / count(sum by(instance, cpu) (node_cpu_seconds_total))
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubprom" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kube-prom"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: kubprom-01
    rules:
      - record: count:up1
@@ -226,17 +174,6 @@ groups:
    rules:
      - record: count:up0
        expr: count without(instance, pod, node) (up == 0)
-EOF
-}
-
-
-resource "aws_prometheus_rule_group_namespace" "apihg" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "api-histogram"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: apihg-01
    rules:
      - record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile
@@ -251,16 +188,6 @@ groups:
        labels:
          quantile: 0.99
          verb: write
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "apibr" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "api-burnrate"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: apibr-01
    rules:
      - record: apiserver_request:burnrate1d
@@ -351,17 +278,6 @@ groups:
        expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[6h])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[6h]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
        labels:
          verb: write
-EOF
-}
-
-
-resource "aws_prometheus_rule_group_namespace" "api-availability" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "api-availability-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: api-01
    rules:
      - record: code_verb:apiserver_request_total:increase30d
@@ -386,16 +302,6 @@ groups:
    rules:
      - record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d
        expr: sum by(cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30)
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubeprom-recording" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubeprom-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
  - name: k8s-01
    rules:
      - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate