💡Setup alertmanager and recording rules

Activate Alertmanager at workspace level, recording rules per workload

examples/workloads.tf

@@ -21,12 +21,14 @@ module "eks_observability_accelerator" {
   create_managed_prometheus_workspace = false
 
   # reusing existing AMP -- needs data source for alerting rules
-  managed_prometheus_id       = var.managed_prometheus_id
+  managed_prometheus_id     = var.managed_prometheus_id
-  managed_prometheus_endpoint = var.managed_prometheus_endpoint
+  managed_prometheus_region = null # defaults to the current region, useful for cross region scenarios
-  managed_prometheus_region   = var.managed_prometheus_region
 
+  # sets up the AMP alert manager at the workspace level
+  enable_alertmanager = true
 
-  enable_java = true
+  enable_java                 = true
+  enable_java_recording_rules = true # defaults to true
 
   # enable_haproxy = true
   # haproxy_config = {

locals.tf

@@ -13,6 +13,11 @@ locals {
   eks_cluster_endpoint = data.aws_eks_cluster.eks_cluster.endpoint
   eks_cluster_version  = data.aws_eks_cluster.eks_cluster.version
 
+  # if region is not passed, we assume the current one
+  amp_ws_region   = coalesce(var.managed_prometheus_region, data.aws_region.current.name)
+  amp_ws_id       = var.create_managed_prometheus_workspace ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_id
+  amp_ws_endpoint = "https://aps-workspaces.${local.amp_ws_region}.amazonaws.com/workspaces/${local.amp_ws_id}/"
+
   context = {
     aws_caller_identity_account_id = data.aws_caller_identity.current.account_id
     aws_caller_identity_arn        = data.aws_caller_identity.current.arn

main.tf

@@ -34,17 +34,32 @@ resource "aws_prometheus_workspace" "this" {
 }
 
 
+resource "aws_prometheus_alert_manager_definition" "this" {
+  count = var.enable_alertmanager ? 1 : 0
+
+  workspace_id = local.amp_ws_id
+
+  # TODO: support custom alert manager config
+  definition = <<EOF
+alertmanager_config: |
+    route:
+      receiver: 'default'
+    receivers:
+      - name: 'default'
+EOF
+}
+
 module "java" {
   count  = var.enable_java ? 1 : 0
   source = "./modules/workloads/java"
 
   addon_context = local.context
 
-  amp_endpoint = var.create_managed_prometheus_workspace ? aws_prometheus_workspace.this[0].prometheus_endpoint : var.managed_prometheus_endpoint
+  amp_endpoint = local.amp_ws_endpoint
-  amp_id       = var.create_managed_prometheus_workspace ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_id
+  amp_id       = local.amp_ws_id
+  amp_region   = local.amp_ws_region
 
-  # if region is not passed, we assume the current one
+  enable_recording_rules = var.enable_java_recording_rules
-  amp_region = try(var.managed_prometheus_region, local.context.aws_region_name)
 
   depends_on = [
     module.operator

modules/workloads/java/main.tf

@@ -7,6 +7,7 @@ locals {
 data "aws_partition" "current" {}
 
 
+# deploys collector
 module "helm_addon" {
   source = "github.com/aws-ia/terraform-aws-eks-blueprints/modules/kubernetes-addons/helm-addon"
 
@@ -63,3 +64,30 @@ module "helm_addon" {
 
   addon_context = var.addon_context
 }
+
+
+resource "aws_prometheus_rule_group_namespace" "this" {
+  count = var.enable_recording_rules ? 1 : 0
+
+  name         = "java_rules"
+  workspace_id = var.amp_id
+  data         = <<EOF
+groups:
+  - name: default-metric
+    rules:
+      - record: metric:recording_rule
+        expr: avg(rate(container_cpu_usage_seconds_total[5m]))
+  - name: default-alert
+    rules:
+      - alert: metric:alerting_rule
+        expr: jvm_memory_bytes_used{job="java", area="heap"} / jvm_memory_bytes_max * 100 > 80
+        for: 1m
+        labels:
+            severity: warning
+        annotations:
+            summary: "JVM heap warning"
+            description: "JVM heap of instance `{{$labels.instance}}` from application `{{$labels.application}}` is above 80% for one minute. (current=`{{$value}}%`)"
+EOF
+}
+
+# dashboard

modules/workloads/java/variables.tf

@@ -1,8 +1,7 @@
-variable "java" {
+variable "enable_recording_rules" {
-  default = {
+  description = "Enable AMP recording rules"
-    a = ""
+  type        = bool
-    b = ""
+  default     = true
-  }
 }
 
 variable "amp_endpoint" {

variables.tf

@@ -55,23 +55,38 @@ variable "managed_prometheus_id" {
   default     = ""
 }
 
-variable "managed_prometheus_endpoint" {
-  description = "AWS Managed Prometheus Workspace endpoint"
-  type        = string
-  default     = ""
-}
 variable "managed_prometheus_region" {
   description = "AWS Managed Prometheus Workspace Region"
   type        = string
   default     = null
 }
 
+variable "enable_alertmanager" {
+  description = "Create AMP AlertManager for all workloads"
+  type        = bool
+  default     = false
+}
+
 variable "enable_java" {
   description = "Deploys a collector for JAVA/JMX based workloads, dashboards and alerting rules"
   type        = bool
   default     = false
 }
 
+variable "enable_java_recording_rules" {
+  description = "Enable AMP recording rules for Java"
+  type        = bool
+  default     = true
+}
+
+# variable "java_config" {
+#   description = "Input configuration Java workloads"
+#   type = object({
+#     enable_recording_rules = bool
+#   })
+# }
+
+
 variable "tags" {
   description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)"
   type        = map(string)