From c3e55fb262ccaa235adabdb51851ef8681f116b1 Mon Sep 17 00:00:00 2001
From: Rodrigue Koffi <bonclay7@users.noreply.github.com>
Date: Mon, 29 Aug 2022 18:55:55 +0200
Subject: [PATCH] Merge alerts and rules in two separate rules

---
 modules/workloads/infra/alerts.tf | 151 +-----------------------------
 modules/workloads/infra/main.tf   |   2 +-
 modules/workloads/infra/rules.tf  |  98 +------------------
 3 files changed, 4 insertions(+), 247 deletions(-)

diff --git a/modules/workloads/infra/alerts.tf b/modules/workloads/infra/alerts.tf
index 78ed8c8..7c3dc0d 100644
--- a/modules/workloads/infra/alerts.tf
+++ b/modules/workloads/infra/alerts.tf
@@ -2,7 +2,7 @@
 # Alerting rules ###############################################################################################################################
 ################################################################################################################################################
 
-resource "aws_prometheus_rule_group_namespace" "nodenw" {
+resource "aws_prometheus_rule_group_namespace" "alerting_rules" {
   count = var.enable_alerting_rules ? 1 : 0
 
   name         = "nodenw-rules"
@@ -19,18 +19,6 @@ groups:
         annotations:
           description: Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}
           summary: Network interface is often changing its status
-EOF
-}
-
-
-
-resource "aws_prometheus_rule_group_namespace" "nodeexporter" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "nodeexporter-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: nodeexp-01
     rules:
       - alert: NodeFilesystemSpaceFillingUp
@@ -208,16 +196,6 @@ groups:
         annotations:
           description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
           summary: Kernel is predicted to exhaust file descriptors limit soon.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyschdlr" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyschdlr-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubesysschdlr-01
     rules:
       - alert: KubeSchedulerDown
@@ -228,16 +206,6 @@ groups:
         annotations:
           description: KubeScheduler has disappeared from Prometheus target discovery.
           summary: Target disappeared from Prometheus target discovery.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyskblt" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyskblt-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubesyskblt-01
     rules:
       - alert: KubeNodeNotReady
@@ -364,17 +332,6 @@ groups:
         annotations:
           description: Kubelet has disappeared from Prometheus target discovery.
           summary: Target disappeared from Prometheus target discovery.
-
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyskbpxy" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyskbpxy-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubesyspxy-01
     rules:
       - alert: KubeProxyDown
@@ -385,16 +342,6 @@ groups:
         annotations:
           description: KubeProxy has disappeared from Prometheus target discovery.
           summary: Target disappeared from Prometheus target discovery.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesys" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesys-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubesys-01
     rules:
       - alert: KubeVersionMismatch
@@ -415,17 +362,6 @@ groups:
         annotations:
           description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
           summary: Kubernetes API server client is experiencing errors.
-
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesyscm" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesyscm-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubesyscm-01
     rules:
       - alert: KubeControllerManagerDown
@@ -436,16 +372,6 @@ groups:
         annotations:
           description: KubeControllerManager has disappeared from Prometheus target discovery.
           summary: Target disappeared from Prometheus target discovery.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubesysapi" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubesysapi-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubesysapi-01
     rules:
       - alert: KubeClientCertificateExpiration
@@ -503,18 +429,6 @@ groups:
         annotations:
           description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
           summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
-EOF
-}
-
-# Default limit of 10 needs to be raised
-
-resource "aws_prometheus_rule_group_namespace" "kubestorage" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubestorage-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubestg-01
     rules:
       - alert: KubePersistentVolumeFillingUp
@@ -565,16 +479,6 @@ groups:
         annotations:
           description: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
           summary: PersistentVolume is having issues with provisioning.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kuberesources" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kuberesources-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kuberes-01
     rules:
       - alert: KubeCPUOvercommit
@@ -655,16 +559,6 @@ groups:
         annotations:
           description: The {{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.
           summary: Processes experience elevated CPU throttling.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubeapps" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubeapps-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubeapps-01
     rules:
       - alert: KubePodCrashLooping
@@ -814,18 +708,6 @@ groups:
         annotations:
           description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.
           summary: HPA is running at max replicas
-EOF
-}
-
-
-
-resource "aws_prometheus_rule_group_namespace" "kubestm" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "kubestm-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubestm-01
     rules:
       - alert: KubeStateMetricsListErrors
@@ -866,16 +748,6 @@ groups:
         annotations:
           description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed.
           summary: kube-state-metrics shards are missing.
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "apislos" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "api-slos"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: apislos-01
     rules:
       - alert: KubeAPIErrorBudgetBurn
@@ -924,17 +796,6 @@ groups:
         annotations:
           description: The API server is burning too much error budget.
           summary: The API server is burning too much error budget.
-EOF
-}
-
-
-resource "aws_prometheus_rule_group_namespace" "generic" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "generic-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: general-01
     rules:
       - alert: TargetDown
@@ -960,16 +821,6 @@ groups:
           severity: none
         annotations:
           description: This is an alert that is used to inhibit info alerts. By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with other alerts. This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a severity of 'warning' or 'critical' starts firing on the same namespace. This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "etcd" {
-  count = var.enable_alerting_rules ? 1 : 0
-
-  name         = "etcd-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: etcd-01
     rules:
       - alert: etcdInsufficientMembers
diff --git a/modules/workloads/infra/main.tf b/modules/workloads/infra/main.tf
index 417ebbd..da97bd8 100644
--- a/modules/workloads/infra/main.tf
+++ b/modules/workloads/infra/main.tf
@@ -44,7 +44,7 @@ resource "helm_release" "prometheus_node_exporter" {
 }
 
 module "helm_addon" {
-  source = "github.com/aws-observability/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
+  source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
 
   helm_config = merge(
     {
diff --git a/modules/workloads/infra/rules.tf b/modules/workloads/infra/rules.tf
index 4a28d93..03fc270 100644
--- a/modules/workloads/infra/rules.tf
+++ b/modules/workloads/infra/rules.tf
@@ -4,10 +4,9 @@
 # Recording rules ##############################################################################################################################
 ################################################################################################################################################
 
-resource "aws_prometheus_rule_group_namespace" "noderules" {
+resource "aws_prometheus_rule_group_namespace" "recording_rules" {
   count = var.enable_recording_rules ? 1 : 0
-
-  name         = "noderules-recording"
+  name         = "infra-recording-rules"
   workspace_id = var.managed_prometheus_workspace_id
   data         = <<EOF
 groups:
@@ -27,16 +26,6 @@ groups:
     rules:
       - record: cluster:node_cpu:ratio_rate5m
         expr: sum(rate(node_cpu_seconds_total{job="node-exporter",mode!="idle",mode!="iowait",mode!="steal"}[5m])) / count(sum by(cluster, instance, cpu) (node_cpu_seconds_total{job="node-exporter"}))
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubelet" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubelet-rules"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubelet-01
     rules:
       - record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
@@ -55,17 +44,6 @@ groups:
         expr: histogram_quantile(0.5, sum by(cluster, instance, le) (rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet",metrics_path="/metrics"})
         labels:
           quantile: 0.5
-
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "ne" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "ne-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: nerules-01
     rules:
       - record: instance:node_num_cpu:sum
@@ -110,16 +88,6 @@ groups:
     rules:
       - record: instance:node_network_transmit_drop_excluding_lo:rate5m
         expr: sum without(device) (rate(node_network_transmit_drop_total{device!="lo",job="node-exporter"}[5m]))
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubescheduler" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubescheduler-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubeschdlr-01
     rules:
       - record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
@@ -174,16 +142,6 @@ groups:
         expr: histogram_quantile(0.5, sum without(instance, pod) (rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])))
         labels:
           quantile: 0.5
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubepromnr" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubeprom-noderecording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubepromnr-01
     rules:
       - record: instance:node_cpu:rate:sum
@@ -208,16 +166,6 @@ groups:
     rules:
       - record: cluster:node_cpu:ratio
         expr: cluster:node_cpu:sum_rate5m / count(sum by(instance, cpu) (node_cpu_seconds_total))
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubprom" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kube-prom"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: kubprom-01
     rules:
       - record: count:up1
@@ -226,17 +174,6 @@ groups:
     rules:
       - record: count:up0
         expr: count without(instance, pod, node) (up == 0)
-EOF
-}
-
-
-resource "aws_prometheus_rule_group_namespace" "apihg" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "api-histogram"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: apihg-01
     rules:
       - record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile
@@ -251,16 +188,6 @@ groups:
         labels:
           quantile: 0.99
           verb: write
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "apibr" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "api-burnrate"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: apibr-01
     rules:
       - record: apiserver_request:burnrate1d
@@ -351,17 +278,6 @@ groups:
         expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[6h])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[6h]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
         labels:
           verb: write
-EOF
-}
-
-
-resource "aws_prometheus_rule_group_namespace" "api-availability" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "api-availability-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: api-01
     rules:
       - record: code_verb:apiserver_request_total:increase30d
@@ -386,16 +302,6 @@ groups:
     rules:
       - record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d
         expr: sum by(cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30)
-EOF
-}
-
-resource "aws_prometheus_rule_group_namespace" "kubeprom-recording" {
-  count = var.enable_recording_rules ? 1 : 0
-
-  name         = "kubeprom-recording"
-  workspace_id = var.managed_prometheus_workspace_id
-  data         = <<EOF
-groups:
   - name: k8s-01
     rules:
       - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate