From 333d46cccca511ba7ec2b83f695324b4c03c342e Mon Sep 17 00:00:00 2001
From: Rodrigue Koffi <bonclay7@users.noreply.github.com>
Date: Wed, 27 Jul 2022 23:37:00 +0200
Subject: [PATCH] =?UTF-8?q?=F0=9F=92=A1Setup=20alertmanager=20and=20record?=
 =?UTF-8?q?ing=20rules?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Activate Alertmanager at workspace level, recording rules per workload
---
 examples/workloads.tf               | 10 ++++++----
 locals.tf                           |  5 +++++
 main.tf                             | 23 +++++++++++++++++++----
 modules/workloads/java/main.tf      | 28 ++++++++++++++++++++++++++++
 modules/workloads/java/variables.tf |  9 ++++-----
 variables.tf                        | 25 ++++++++++++++++++++-----
 6 files changed, 82 insertions(+), 18 deletions(-)

diff --git a/examples/workloads.tf b/examples/workloads.tf
index 7c1e212..2334d02 100644
--- a/examples/workloads.tf
+++ b/examples/workloads.tf
@@ -21,12 +21,14 @@ module "eks_observability_accelerator" {
   create_managed_prometheus_workspace = false
 
   # reusing existing AMP -- needs data source for alerting rules
-  managed_prometheus_id       = var.managed_prometheus_id
-  managed_prometheus_endpoint = var.managed_prometheus_endpoint
-  managed_prometheus_region   = var.managed_prometheus_region
+  managed_prometheus_id     = var.managed_prometheus_id
+  managed_prometheus_region = null # defaults to the current region, useful for cross region scenarios
 
+  # sets up the AMP alert manager at the workspace level
+  enable_alertmanager = true
 
-  enable_java = true
+  enable_java                 = true
+  enable_java_recording_rules = true # defaults to true
 
   # enable_haproxy = true
   # haproxy_config = {
diff --git a/locals.tf b/locals.tf
index 46aa6b9..be34fe5 100644
--- a/locals.tf
+++ b/locals.tf
@@ -13,6 +13,11 @@ locals {
   eks_cluster_endpoint = data.aws_eks_cluster.eks_cluster.endpoint
   eks_cluster_version  = data.aws_eks_cluster.eks_cluster.version
 
+  # if region is not passed, we assume the current one
+  amp_ws_region   = coalesce(var.managed_prometheus_region, data.aws_region.current.name)
+  amp_ws_id       = var.create_managed_prometheus_workspace ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_id
+  amp_ws_endpoint = "https://aps-workspaces.${local.amp_ws_region}.amazonaws.com/workspaces/${local.amp_ws_id}/"
+
   context = {
     aws_caller_identity_account_id = data.aws_caller_identity.current.account_id
     aws_caller_identity_arn        = data.aws_caller_identity.current.arn
diff --git a/main.tf b/main.tf
index b9c50f8..962cce6 100644
--- a/main.tf
+++ b/main.tf
@@ -34,17 +34,32 @@ resource "aws_prometheus_workspace" "this" {
 }
 
 
+resource "aws_prometheus_alert_manager_definition" "this" {
+  count = var.enable_alertmanager ? 1 : 0
+
+  workspace_id = local.amp_ws_id
+
+  # TODO: support custom alert manager config
+  definition = <<EOF
+alertmanager_config: |
+    route:
+      receiver: 'default'
+    receivers:
+      - name: 'default'
+EOF
+}
+
 module "java" {
   count  = var.enable_java ? 1 : 0
   source = "./modules/workloads/java"
 
   addon_context = local.context
 
-  amp_endpoint = var.create_managed_prometheus_workspace ? aws_prometheus_workspace.this[0].prometheus_endpoint : var.managed_prometheus_endpoint
-  amp_id       = var.create_managed_prometheus_workspace ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_id
+  amp_endpoint = local.amp_ws_endpoint
+  amp_id       = local.amp_ws_id
+  amp_region   = local.amp_ws_region
 
-  # if region is not passed, we assume the current one
-  amp_region = try(var.managed_prometheus_region, local.context.aws_region_name)
+  enable_recording_rules = var.enable_java_recording_rules
 
   depends_on = [
     module.operator
diff --git a/modules/workloads/java/main.tf b/modules/workloads/java/main.tf
index 4c733c1..623a147 100644
--- a/modules/workloads/java/main.tf
+++ b/modules/workloads/java/main.tf
@@ -7,6 +7,7 @@ locals {
 data "aws_partition" "current" {}
 
 
+# deploys collector
 module "helm_addon" {
   source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
 
@@ -63,3 +64,30 @@ module "helm_addon" {
 
   addon_context = var.addon_context
 }
+
+
+resource "aws_prometheus_rule_group_namespace" "this" {
+  count = var.enable_recording_rules ? 1 : 0
+
+  name         = "java_rules"
+  workspace_id = var.amp_id
+  data         = <<EOF
+groups:
+  - name: default-metric
+    rules:
+      - record: metric:recording_rule
+        expr: avg(rate(container_cpu_usage_seconds_total[5m]))
+  - name: default-alert
+    rules:
+      - alert: metric:alerting_rule
+        expr: jvm_memory_bytes_used{job="java", area="heap"} / jvm_memory_bytes_max * 100 > 80
+        for: 1m
+        labels:
+            severity: warning
+        annotations:
+            summary: "JVM heap warning"
+            description: "JVM heap of instance `{{$labels.instance}}` from application `{{$labels.application}}` is above 80% for one minute. (current=`{{$value}}%`)"
+EOF
+}
+
+# dashboard
diff --git a/modules/workloads/java/variables.tf b/modules/workloads/java/variables.tf
index 272d366..9315317 100644
--- a/modules/workloads/java/variables.tf
+++ b/modules/workloads/java/variables.tf
@@ -1,8 +1,7 @@
-variable "java" {
-  default = {
-    a = ""
-    b = ""
-  }
+variable "enable_recording_rules" {
+  description = "Enable AMP recording rules"
+  type        = bool
+  default     = true
 }
 
 variable "amp_endpoint" {
diff --git a/variables.tf b/variables.tf
index a920985..01ee50e 100644
--- a/variables.tf
+++ b/variables.tf
@@ -55,23 +55,38 @@ variable "managed_prometheus_id" {
   default     = ""
 }
 
-variable "managed_prometheus_endpoint" {
-  description = "AWS Managed Prometheus Workspace endpoint"
-  type        = string
-  default     = ""
-}
 variable "managed_prometheus_region" {
   description = "AWS Managed Prometheus Workspace Region"
   type        = string
   default     = null
 }
 
+variable "enable_alertmanager" {
+  description = "Create AMP AlertManager for all workloads"
+  type        = bool
+  default     = false
+}
+
 variable "enable_java" {
   description = "Deploys a collector for JAVA/JMX based workloads, dashboards and alerting rules"
   type        = bool
   default     = false
 }
 
+variable "enable_java_recording_rules" {
+  description = "Enable AMP recording rules for Java"
+  type        = bool
+  default     = true
+}
+
+# variable "java_config" {
+#   description = "Input configuration Java workloads"
+#   type = object({
+#     enable_recording_rules = bool
+#   })
+# }
+
+
 variable "tags" {
   description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)"
   type        = map(string)