mirror of
https://github.com/storytold/terraform-aws-observability-accelerator.git
synced 2026-10-09 00:09:43 +00:00
Compose EKS monitoring modules (#115)
* Move modules around * Update amp billing source * Merge Java monitoring to EKS * Update docs * Merge nginx pattern * Pre-commit * Add save and test URL output * Move EKS dependencies to EKS monitoring module * update docs * Update examples and docs * Add java doc * Add NGINX doc * Update nginx doc * Fix amp monitoring example path * Fix pre-commit * Todo: move to main after merge * Update docs, fix tags
This commit is contained in:
Rodrigue Koffi
@@ -0,0 +1,97 @@
|
||||
# Infrastructure monitoring
|
||||
|
||||
This module provides EKS cluster monitoring with the following resources:
|
||||
|
||||
- AWS Distro For OpenTelemetry Operator and Collector
|
||||
- AWS Managed Grafana Dashboard and data source
|
||||
- Alerts and recording rules with AWS Managed Service for Prometheus
|
||||
|
||||
This module is inspired from the open source [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
|
||||
|
||||
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
## Requirements
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
|
||||
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
|
||||
| <a name="requirement_grafana"></a> [grafana](#requirement\_grafana) | >= 1.25.0 |
|
||||
| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.4.1 |
|
||||
| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14 |
|
||||
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
|
||||
| <a name="provider_grafana"></a> [grafana](#provider\_grafana) | >= 1.25.0 |
|
||||
| <a name="provider_helm"></a> [helm](#provider\_helm) | >= 2.4.1 |
|
||||
|
||||
## Modules
|
||||
|
||||
| Name | Source | Version |
|
||||
|------|--------|---------|
|
||||
| <a name="module_helm_addon"></a> [helm\_addon](#module\_helm\_addon) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.13.1 |
|
||||
| <a name="module_java_monitoring"></a> [java\_monitoring](#module\_java\_monitoring) | ./patterns/java | n/a |
|
||||
| <a name="module_nginx_monitoring"></a> [nginx\_monitoring](#module\_nginx\_monitoring) | ./patterns/nginx | n/a |
|
||||
| <a name="module_operator"></a> [operator](#module\_operator) | ./add-ons/adot-operator | n/a |
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|------|------|
|
||||
| [aws_prometheus_rule_group_namespace.alerting_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource |
|
||||
| [aws_prometheus_rule_group_namespace.recording_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource |
|
||||
| [grafana_dashboard.cluster](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
| [grafana_dashboard.kubelet](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
| [grafana_dashboard.nodeexp_nodes](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
| [grafana_dashboard.nodes](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
| [grafana_dashboard.nsworkload](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
| [grafana_dashboard.workloads](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
| [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
|
||||
| [helm_release.prometheus_node_exporter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
|
||||
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
|
||||
| [aws_eks_cluster.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
|
||||
| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
|
||||
| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|---------|:--------:|
|
||||
| <a name="input_custom_metrics_config"></a> [custom\_metrics\_config](#input\_custom\_metrics\_config) | Configuration object to enable custom metrics collection | <pre>object({<br> ports = list(number)<br> # paths = optional(list(string), ["/metrics"])<br> # list of samples to be dropped by label prefix, ex: go_ -> discards go_.*<br> dropped_series_prefixes = list(string)<br> })</pre> | <pre>{<br> "dropped_series_prefixes": [<br> "unspecified"<br> ],<br> "ports": []<br>}</pre> | no |
|
||||
| <a name="input_dashboards_folder_id"></a> [dashboards\_folder\_id](#input\_dashboards\_folder\_id) | Grafana folder ID for automatic dashboards | `string` | n/a | yes |
|
||||
| <a name="input_eks_cluster_id"></a> [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes |
|
||||
| <a name="input_enable_alerting_rules"></a> [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no |
|
||||
| <a name="input_enable_amazon_eks_adot"></a> [enable\_amazon\_eks\_adot](#input\_enable\_amazon\_eks\_adot) | Enables the ADOT Operator on the EKS Cluster | `bool` | `true` | no |
|
||||
| <a name="input_enable_cert_manager"></a> [enable\_cert\_manager](#input\_enable\_cert\_manager) | Allow reusing an existing installation of cert-manager | `bool` | `true` | no |
|
||||
| <a name="input_enable_custom_metrics"></a> [enable\_custom\_metrics](#input\_enable\_custom\_metrics) | Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included | `bool` | `false` | no |
|
||||
| <a name="input_enable_dashboards"></a> [enable\_dashboards](#input\_enable\_dashboards) | Enables or disables curated dashboards | `bool` | `true` | no |
|
||||
| <a name="input_enable_java"></a> [enable\_java](#input\_enable\_java) | Enable Java workloads monitoring, alerting and default dashboards | `bool` | `false` | no |
|
||||
| <a name="input_enable_kube_state_metrics"></a> [enable\_kube\_state\_metrics](#input\_enable\_kube\_state\_metrics) | Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
|
||||
| <a name="input_enable_nginx"></a> [enable\_nginx](#input\_enable\_nginx) | Enable NGINX workloads monitoring, alerting and default dashboards | `bool` | `false` | no |
|
||||
| <a name="input_enable_node_exporter"></a> [enable\_node\_exporter](#input\_enable\_node\_exporter) | Enables or disables Node exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
|
||||
| <a name="input_enable_tracing"></a> [enable\_tracing](#input\_enable\_tracing) | (Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0 | `bool` | `false` | no |
|
||||
| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no |
|
||||
| <a name="input_irsa_iam_permissions_boundary"></a> [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `null` | no |
|
||||
| <a name="input_irsa_iam_role_path"></a> [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no |
|
||||
| <a name="input_java_config"></a> [java\_config](#input\_java\_config) | Configuration object for Java/JMX monitoring | <pre>object({<br> enable_alerting_rules = bool<br> scrape_sample_limit = number<br> })</pre> | <pre>{<br> "enable_alerting_rules": true,<br> "scrape_sample_limit": 1000<br>}</pre> | no |
|
||||
| <a name="input_ksm_config"></a> [ksm\_config](#input\_ksm\_config) | Kube State metrics configuration | <pre>object({<br> create_namespace = bool<br> k8s_namespace = string<br> helm_chart_name = string<br> helm_chart_version = string<br> helm_release_name = string<br> helm_repo_url = string<br> helm_settings = map(string)<br> helm_values = map(any)<br><br> scrape_interval = string<br> scrape_timeout = string<br> })</pre> | <pre>{<br> "create_namespace": true,<br> "helm_chart_name": "kube-state-metrics",<br> "helm_chart_version": "4.24.0",<br> "helm_release_name": "kube-state-metrics",<br> "helm_repo_url": "https://prometheus-community.github.io/helm-charts",<br> "helm_settings": {},<br> "helm_values": {},<br> "k8s_namespace": "kube-system",<br> "scrape_interval": "60s",<br> "scrape_timeout": "15s"<br>}</pre> | no |
|
||||
| <a name="input_managed_prometheus_workspace_endpoint"></a> [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no |
|
||||
| <a name="input_managed_prometheus_workspace_id"></a> [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no |
|
||||
| <a name="input_managed_prometheus_workspace_region"></a> [managed\_prometheus\_workspace\_region](#input\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus Workspace's Region | `string` | `null` | no |
|
||||
| <a name="input_ne_config"></a> [ne\_config](#input\_ne\_config) | Node exporter configuration | <pre>object({<br> create_namespace = bool<br> k8s_namespace = string<br> helm_chart_name = string<br> helm_chart_version = string<br> helm_release_name = string<br> helm_repo_url = string<br> helm_settings = map(string)<br> helm_values = map(any)<br><br> scrape_interval = string<br> scrape_timeout = string<br> })</pre> | <pre>{<br> "create_namespace": true,<br> "helm_chart_name": "prometheus-node-exporter",<br> "helm_chart_version": "2.0.3",<br> "helm_release_name": "prometheus-node-exporter",<br> "helm_repo_url": "https://prometheus-community.github.io/helm-charts",<br> "helm_settings": {},<br> "helm_values": {},<br> "k8s_namespace": "prometheus-node-exporter",<br> "scrape_interval": "60s",<br> "scrape_timeout": "60s"<br>}</pre> | no |
|
||||
| <a name="input_nginx_config"></a> [nginx\_config](#input\_nginx\_config) | Configuration object for NGINX monitoring | <pre>object({<br> enable_alerting_rules = bool<br> scrape_sample_limit = number<br> prometheus_metrics_endpoint = string<br> })</pre> | <pre>{<br> "enable_alerting_rules": true,<br> "prometheus_metrics_endpoint": "metrics",<br> "scrape_sample_limit": 1000<br>}</pre> | no |
|
||||
| <a name="input_prometheus_config"></a> [prometheus\_config](#input\_prometheus\_config) | Controls default values such as scrape interval, timeouts and ports globally | <pre>object({<br> global_scrape_interval = string<br> global_scrape_timeout = string<br> })</pre> | <pre>{<br> "global_scrape_interval": "60s",<br> "global_scrape_timeout": "15s"<br>}</pre> | no |
|
||||
| <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no |
|
||||
| <a name="input_tracing_config"></a> [tracing\_config](#input\_tracing\_config) | Configuration object for traces collection to AWS X-Ray | <pre>object({<br> otlp_grpc_endpoint = string<br> otlp_http_endpoint = string<br> send_batch_size = number<br> timeout = string<br> })</pre> | <pre>{<br> "otlp_grpc_endpoint": "0.0.0.0:4317",<br> "otlp_http_endpoint": "0.0.0.0:4318",<br> "send_batch_size": 50,<br> "timeout": "30s"<br>}</pre> | no |
|
||||
|
||||
## Outputs
|
||||
|
||||
| Name | Description |
|
||||
|------|-------------|
|
||||
| <a name="output_eks_cluster_id"></a> [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id |
|
||||
| <a name="output_eks_cluster_version"></a> [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version |
|
||||
| <a name="output_grafana_dashboard_urls"></a> [grafana\_dashboard\_urls](#output\_grafana\_dashboard\_urls) | URLs for dashboards created |
|
||||
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
@@ -0,0 +1,71 @@
|
||||
# AWS OpenTelemetry Operator
|
||||
|
||||
[AWS Distro for OpenTelemetry (ADOT)](https://aws-otel.github.io/) is a secure,
|
||||
production-ready, AWS-supported distribution of the OpenTelemetry project.
|
||||
Part of the Cloud Native Computing Foundation, OpenTelemetry provides open
|
||||
source APIs, libraries, and agents to collect distributed traces and metrics
|
||||
for application monitoring.
|
||||
|
||||
This modules deploys either the
|
||||
[AWS Managed ADOT OpenTelemetry Operator for EKS](https://aws.amazon.com/about-aws/whats-new/2022/04/eks-opentelemetry-operator-now-available/)
|
||||
or [the OpenTelemetry Operator](https://github.com/open-telemetry/opentelemetry-helm-charts)
|
||||
through helm.
|
||||
The OpenTelemetry Operator is an implementation of a Kubernetes Operator.
|
||||
A Kubernetes Operator is a method of packaging, deploying and managing a
|
||||
Kubernetes-native application, which is both deployed on Kubernetes and
|
||||
managed using the Kubernetes APIs and kubectl tooling. The Kubernetes Operator
|
||||
is a custom controller, which introduces new object types through Custom Resource
|
||||
Definition (CRD), an extension mechanism in Kubernetes.
|
||||
In this case, the CRD that is managed by the OpenTelemetry Operator is the Collector.
|
||||
|
||||
> :warning: We do install [cert-manager](https://cert-manager.io/) as a [hard requirement](https://docs.aws.amazon.com/eks/latest/userguide/opentelemetry.html) for
|
||||
the ADOT Operator.
|
||||
|
||||
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
## Requirements
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
|
||||
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
|
||||
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
|
||||
| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | >= 2.10 |
|
||||
|
||||
## Modules
|
||||
|
||||
| Name | Source | Version |
|
||||
|------|--------|---------|
|
||||
| <a name="module_cert_manager"></a> [cert\_manager](#module\_cert\_manager) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/cert-manager | v4.13.1 |
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|------|------|
|
||||
| [aws_eks_addon.adot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource |
|
||||
| [kubernetes_cluster_role_binding_v1.adot](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding_v1) | resource |
|
||||
| [kubernetes_cluster_role_v1.adot](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_v1) | resource |
|
||||
| [kubernetes_namespace_v1.adot](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |
|
||||
| [kubernetes_role_binding_v1.adot](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding_v1) | resource |
|
||||
| [kubernetes_role_v1.adot](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_v1) | resource |
|
||||
| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|---------|:--------:|
|
||||
| <a name="input_addon_config"></a> [addon\_config](#input\_addon\_config) | Amazon EKS Managed ADOT Add-on config | `any` | `{}` | no |
|
||||
| <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br> aws_caller_identity_account_id = string<br> aws_caller_identity_arn = string<br> aws_eks_cluster_endpoint = string<br> aws_partition_id = string<br> aws_region_name = string<br> eks_cluster_id = string<br> eks_oidc_issuer_url = string<br> eks_oidc_provider_arn = string<br> irsa_iam_role_path = string<br> irsa_iam_permissions_boundary = string<br> tags = map(string)<br> })</pre> | n/a | yes |
|
||||
| <a name="input_enable_cert_manager"></a> [enable\_cert\_manager](#input\_enable\_cert\_manager) | Enable cert-manager, a requirement for ADOT Operator | `bool` | `true` | no |
|
||||
| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm provider config for cert-manager | `any` | <pre>{<br> "version": "v1.8.2"<br>}</pre> | no |
|
||||
| <a name="input_kubernetes_version"></a> [kubernetes\_version](#input\_kubernetes\_version) | EKS Cluster version | `string` | n/a | yes |
|
||||
|
||||
## Outputs
|
||||
|
||||
No outputs.
|
||||
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
@@ -0,0 +1,6 @@
|
||||
locals {
|
||||
name = "adot"
|
||||
eks_addon_role_name = "eks:addon-manager"
|
||||
eks_addon_clusterrole_name = "eks:addon-manager-otel"
|
||||
addon_namespace = "opentelemetry-operator-system"
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
module "cert_manager" {
|
||||
source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/cert-manager?ref=v4.13.1"
|
||||
count = var.enable_cert_manager ? 1 : 0
|
||||
|
||||
helm_config = var.helm_config
|
||||
addon_context = var.addon_context
|
||||
}
|
||||
|
||||
resource "kubernetes_namespace_v1" "adot" {
|
||||
|
||||
metadata {
|
||||
# If using EKS addon, namespace must be "opentelemetry-operator-system"
|
||||
name = local.addon_namespace
|
||||
|
||||
labels = {
|
||||
# Prerequisite for EKS addon
|
||||
"control-plane" = "controller-manager"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_eks_addon_version" "this" {
|
||||
addon_name = local.name
|
||||
kubernetes_version = try(var.addon_config.kubernetes_version, var.kubernetes_version)
|
||||
most_recent = try(var.addon_config.most_recent, true)
|
||||
}
|
||||
|
||||
resource "aws_eks_addon" "adot" {
|
||||
cluster_name = var.addon_context.eks_cluster_id
|
||||
addon_name = local.name
|
||||
addon_version = try(var.addon_config.addon_version, data.aws_eks_addon_version.this.version)
|
||||
resolve_conflicts = try(var.addon_config.resolve_conflicts, "OVERWRITE")
|
||||
service_account_role_arn = try(var.addon_config.service_account_role_arn, null)
|
||||
preserve = try(var.addon_config.preserve, true)
|
||||
|
||||
tags = merge(
|
||||
var.addon_context.tags,
|
||||
try(var.addon_config.tags, {}),
|
||||
# implicit dependency with roles
|
||||
{
|
||||
RoleVersion = try(kubernetes_role_v1.adot.metadata[0].resource_version, ""),
|
||||
ClusterRoleVersion = try(kubernetes_cluster_role_v1.adot.metadata[0].resource_version, "")
|
||||
}
|
||||
)
|
||||
|
||||
depends_on = [module.cert_manager]
|
||||
}
|
||||
|
||||
resource "kubernetes_role_v1" "adot" {
|
||||
|
||||
metadata {
|
||||
name = local.eks_addon_role_name
|
||||
namespace = kubernetes_namespace_v1.adot.metadata[0].name
|
||||
}
|
||||
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["serviceaccounts"]
|
||||
resource_names = ["opentelemetry-operator-controller-manager"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["rbac.authorization.k8s.io"]
|
||||
resources = ["roles"]
|
||||
resource_names = ["opentelemetry-operator-leader-election-role"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["rbac.authorization.k8s.io"]
|
||||
resources = ["rolebindings"]
|
||||
resource_names = ["opentelemetry-operator-leader-election-rolebinding"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["services"]
|
||||
resource_names = ["opentelemetry-operator-controller-manager-metrics-service", "opentelemetry-operator-webhook-service"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["apps"]
|
||||
resources = ["deployments"]
|
||||
resource_names = ["opentelemetry-operator-controller-manager"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["cert-manager.io"]
|
||||
resources = ["certificates", "issuers"]
|
||||
resource_names = ["opentelemetry-operator-serving-cert", "opentelemetry-operator-selfsigned-issuer"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["configmaps"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["configmaps/status"]
|
||||
verbs = ["get", "update", "patch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["events"]
|
||||
verbs = ["create", "patch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["pods"]
|
||||
verbs = ["list"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_role_binding_v1" "adot" {
|
||||
|
||||
metadata {
|
||||
name = local.eks_addon_role_name
|
||||
namespace = kubernetes_namespace_v1.adot.metadata[0].name
|
||||
}
|
||||
|
||||
subject {
|
||||
kind = "User"
|
||||
name = local.eks_addon_role_name
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
}
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "Role"
|
||||
name = local.eks_addon_role_name
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cluster_role_v1" "adot" {
|
||||
|
||||
metadata {
|
||||
name = local.eks_addon_clusterrole_name
|
||||
}
|
||||
|
||||
rule {
|
||||
api_groups = ["apiextensions.k8s.io"]
|
||||
resources = ["customresourcedefinitions"]
|
||||
resource_names = ["opentelemetrycollectors.opentelemetry.io", "instrumentations.opentelemetry.io"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["namespaces"]
|
||||
resource_names = [kubernetes_namespace_v1.adot.metadata[0].name]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["rbac.authorization.k8s.io"]
|
||||
resources = ["clusterroles"]
|
||||
resource_names = ["opentelemetry-operator-manager-role", "opentelemetry-operator-metrics-reader", "opentelemetry-operator-proxy-role"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["rbac.authorization.k8s.io"]
|
||||
resources = ["clusterrolebindings"]
|
||||
resource_names = ["opentelemetry-operator-manager-rolebinding", "opentelemetry-operator-proxy-rolebinding"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["admissionregistration.k8s.io"]
|
||||
resources = ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
|
||||
resource_names = ["opentelemetry-operator-mutating-webhook-configuration", "opentelemetry-operator-validating-webhook-configuration"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["networking.k8s.io"]
|
||||
resources = ["ingresses"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
non_resource_urls = ["/metrics"]
|
||||
verbs = ["get"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["configmaps"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["events"]
|
||||
verbs = ["create", "patch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["namespaces"]
|
||||
verbs = ["list", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["serviceaccounts"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["services"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["apps"]
|
||||
resources = ["daemonsets"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["apps"]
|
||||
resources = ["deployments"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["apps"]
|
||||
resources = ["replicasets"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["apps"]
|
||||
resources = ["statefulsets"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["autoscaling"]
|
||||
resources = ["horizontalpodautoscalers"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["coordination.k8s.io"]
|
||||
resources = ["leases"]
|
||||
verbs = ["create", "get", "list", "update"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["opentelemetry.io"]
|
||||
resources = ["opentelemetrycollectors"]
|
||||
verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["opentelemetry.io"]
|
||||
resources = ["opentelemetrycollectors/finalizers"]
|
||||
verbs = ["get", "patch", "update"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["opentelemetry.io"]
|
||||
resources = ["opentelemetrycollectors/status"]
|
||||
verbs = ["get", "patch", "update"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["opentelemetry.io"]
|
||||
resources = ["instrumentations"]
|
||||
verbs = ["get", "list", "patch", "update", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["authentication.k8s.io"]
|
||||
resources = ["tokenreviews"]
|
||||
verbs = ["create"]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["authorization.k8s.io"]
|
||||
resources = ["subjectaccessreviews"]
|
||||
verbs = ["create"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cluster_role_binding_v1" "adot" {
|
||||
|
||||
metadata {
|
||||
name = local.eks_addon_clusterrole_name
|
||||
}
|
||||
subject {
|
||||
kind = "User"
|
||||
name = local.eks_addon_role_name
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
}
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = local.eks_addon_clusterrole_name
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
variable "helm_config" {
|
||||
description = "Helm provider config for cert-manager"
|
||||
type = any
|
||||
default = { version = "v1.8.2" }
|
||||
}
|
||||
|
||||
variable "addon_context" {
|
||||
description = "Input configuration for the addon"
|
||||
type = object({
|
||||
aws_caller_identity_account_id = string
|
||||
aws_caller_identity_arn = string
|
||||
aws_eks_cluster_endpoint = string
|
||||
aws_partition_id = string
|
||||
aws_region_name = string
|
||||
eks_cluster_id = string
|
||||
eks_oidc_issuer_url = string
|
||||
eks_oidc_provider_arn = string
|
||||
irsa_iam_role_path = string
|
||||
irsa_iam_permissions_boundary = string
|
||||
tags = map(string)
|
||||
})
|
||||
}
|
||||
|
||||
variable "enable_cert_manager" {
|
||||
description = "Enable cert-manager, a requirement for ADOT Operator"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "kubernetes_version" {
|
||||
description = "EKS Cluster version"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "addon_config" {
|
||||
description = "Amazon EKS Managed ADOT Add-on config"
|
||||
type = any
|
||||
default = {}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
terraform {
|
||||
required_version = ">= 1.1.0"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 3.72"
|
||||
}
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = ">= 2.10"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,755 @@
|
||||
################################################################################################################################################
|
||||
# Alerting rules ###############################################################################################################################
|
||||
################################################################################################################################################
|
||||
|
||||
resource "aws_prometheus_rule_group_namespace" "alerting_rules" {
|
||||
count = var.enable_alerting_rules ? 1 : 0
|
||||
|
||||
name = "accelerator-infra-alerting"
|
||||
workspace_id = var.managed_prometheus_workspace_id
|
||||
data = <<EOF
|
||||
groups:
|
||||
- name: infra-alerts-01
|
||||
rules:
|
||||
- alert: NodeNetworkInterfaceFlapping
|
||||
expr: changes(node_network_up{device!~"veth.+",job="node-exporter"}[2m]) > 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}
|
||||
summary: Network interface is often changing its status
|
||||
- alert: NodeFilesystemSpaceFillingUp
|
||||
expr: (node_filesystem_avail_bytes{fstype!="",job="node-exporter"} / node_filesystem_size_bytes{fstype!="",job="node-exporter"} * 100 < 15 and predict_linear(node_filesystem_avail_bytes{fstype!="",job="node-exporter"}[6h], 24 * 60 * 60) < 0 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
|
||||
summary: Filesystem is predicted to run out of space within the next 24 hours.
|
||||
- alert: NodeFilesystemSpaceFillingUp
|
||||
expr: (node_filesystem_avail_bytes{fstype!="",job="node-exporter"} / node_filesystem_size_bytes{fstype!="",job="node-exporter"} * 100 < 10 and predict_linear(node_filesystem_avail_bytes{fstype!="",job="node-exporter"}[6h], 4 * 60 * 60) < 0 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
|
||||
summary: Filesystem is predicted to run out of space within the next 4 hours.
|
||||
- alert: NodeFilesystemAlmostOutOfSpace
|
||||
expr: (node_filesystem_avail_bytes{fstype!="",job="node-exporter"} / node_filesystem_size_bytes{fstype!="",job="node-exporter"} * 100 < 3 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
|
||||
summary: Filesystem has less than 3% space left.
|
||||
- alert: NodeFilesystemAlmostOutOfSpace
|
||||
expr: (node_filesystem_avail_bytes{fstype!="",job="node-exporter"} / node_filesystem_size_bytes{fstype!="",job="node-exporter"} * 100 < 5 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
|
||||
summary: Filesystem has less than 5% space left.
|
||||
- alert: NodeFilesystemFilesFillingUp
|
||||
expr: (node_filesystem_files_free{fstype!="",job="node-exporter"} / node_filesystem_files{fstype!="",job="node-exporter"} * 100 < 40 and predict_linear(node_filesystem_files_free{fstype!="",job="node-exporter"}[6h], 24 * 60 * 60) < 0 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
|
||||
summary: Filesystem is predicted to run out of inodes within the next 24 hours.
|
||||
- alert: NodeFilesystemFilesFillingUp
|
||||
expr: (node_filesystem_files_free{fstype!="",job="node-exporter"} / node_filesystem_files{fstype!="",job="node-exporter"} * 100 < 20 and predict_linear(node_filesystem_files_free{fstype!="",job="node-exporter"}[6h], 4 * 60 * 60) < 0 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
|
||||
summary: Filesystem is predicted to run out of inodes within the next 4 hours.
|
||||
- alert: NodeFilesystemAlmostOutOfFiles
|
||||
expr: (node_filesystem_files_free{fstype!="",job="node-exporter"} / node_filesystem_files{fstype!="",job="node-exporter"} * 100 < 5 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
|
||||
summary: Filesystem has less than 5% inodes left.
|
||||
- alert: NodeFilesystemAlmostOutOfFiles
|
||||
expr: (node_filesystem_files_free{fstype!="",job="node-exporter"} / node_filesystem_files{fstype!="",job="node-exporter"} * 100 < 3 and node_filesystem_readonly{fstype!="",job="node-exporter"} == 0)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
|
||||
summary: Filesystem has less than 3% inodes left.
|
||||
- alert: NodeNetworkReceiveErrs
|
||||
expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.
|
||||
summary: Network interface is reporting many receive errors.
|
||||
- alert: NodeNetworkTransmitErrs
|
||||
expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.
|
||||
summary: Network interface is reporting many transmit errors.
|
||||
- alert: NodeHighNumberConntrackEntriesUsed
|
||||
expr: (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $value | humanizePercentage }} of conntrack entries are used.
|
||||
summary: Number of conntrack are getting close to the limit.
|
||||
- alert: NodeTextFileCollectorScrapeError
|
||||
expr: node_textfile_scrape_error{job="node-exporter"} == 1
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Node Exporter text file collector failed to scrape.
|
||||
summary: Node Exporter text file collector failed to scrape.
|
||||
- alert: NodeClockSkewDetected
|
||||
expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.
|
||||
summary: Clock skew detected.
|
||||
- alert: NodeClockNotSynchronising
|
||||
expr: min_over_time(node_timex_sync_status[5m]) == 0 and node_timex_maxerror_seconds >= 16
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.
|
||||
summary: Clock not synchronising.
|
||||
- alert: NodeRAIDDegraded
|
||||
expr: node_md_disks_required - ignoring(state) (node_md_disks{state="active"}) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: RAID array '{{ $labels.device }}' on {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
|
||||
summary: RAID Array is degraded
|
||||
- alert: NodeRAIDDiskFailure
|
||||
expr: node_md_disks{state="failed"} > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.
|
||||
summary: Failed device in RAID array
|
||||
- alert: NodeFileDescriptorLimit
|
||||
expr: (node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
|
||||
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
||||
- alert: NodeFileDescriptorLimit
|
||||
expr: (node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
|
||||
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
||||
- alert: KubeSchedulerDown
|
||||
expr: absent(up{job="kube-scheduler"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: KubeScheduler has disappeared from Prometheus target discovery.
|
||||
summary: Target disappeared from Prometheus target discovery.
|
||||
- name: infra-alerts-02
|
||||
rules:
|
||||
- alert: KubeNodeNotReady
|
||||
expr: kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} == 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $labels.node }} has been unready for more than 15 minutes.
|
||||
summary: Node is not ready.
|
||||
- alert: KubeNodeUnreachable
|
||||
expr: (kube_node_spec_taint{effect="NoSchedule",job="kube-state-metrics",key="node.kubernetes.io/unreachable"} unless ignoring(key, value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $labels.node }} is unreachable and some workloads may be rescheduled.
|
||||
summary: Node is unreachable.
|
||||
- alert: KubeletTooManyPods
|
||||
expr: count by(cluster, node) ((kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance, pod, namespace, cluster) group_left(node) topk by(instance, pod, namespace, cluster) (1, kube_pod_info{job="kube-state-metrics"})) / max by(cluster, node) (kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1) > 0.95
|
||||
for: 15m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
|
||||
summary: Kubelet is running at capacity.
|
||||
- alert: KubeNodeReadinessFlapping
|
||||
expr: sum by(cluster, node) (changes(kube_node_status_condition{condition="Ready",status="true"}[15m])) > 2
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
|
||||
summary: Node readiness status is flapping.
|
||||
- alert: KubeletPlegDurationHigh
|
||||
expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
|
||||
summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
|
||||
- alert: KubeletPodStartUpLatencyHigh
|
||||
expr: histogram_quantile(0.99, sum by(cluster, instance, le) (rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m]))) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet"} > 60
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
|
||||
summary: Kubelet Pod startup latency is too high.
|
||||
- alert: KubeletClientCertificateExpiration
|
||||
expr: kubelet_certificate_manager_client_ttl_seconds < 604800
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
|
||||
summary: Kubelet client certificate is about to expire.
|
||||
- alert: KubeletClientCertificateExpiration
|
||||
expr: kubelet_certificate_manager_client_ttl_seconds < 86400
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
|
||||
summary: Kubelet client certificate is about to expire.
|
||||
- alert: KubeletServerCertificateExpiration
|
||||
expr: kubelet_certificate_manager_server_ttl_seconds < 604800
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
|
||||
summary: Kubelet server certificate is about to expire.
|
||||
- alert: KubeletServerCertificateExpiration
|
||||
expr: kubelet_certificate_manager_server_ttl_seconds < 86400
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
|
||||
summary: Kubelet server certificate is about to expire.
|
||||
- alert: KubeletClientCertificateRenewalErrors
|
||||
expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).
|
||||
summary: Kubelet has failed to renew its client certificate.
|
||||
- alert: KubeletServerCertificateRenewalErrors
|
||||
expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
|
||||
summary: Kubelet has failed to renew its server certificate.
|
||||
- alert: KubeletDown
|
||||
expr: absent(up{job="kubelet"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: Kubelet has disappeared from Prometheus target discovery.
|
||||
summary: Target disappeared from Prometheus target discovery.
|
||||
- alert: KubeProxyDown
|
||||
expr: absent(up{job="kube-proxy"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: KubeProxy has disappeared from Prometheus target discovery.
|
||||
summary: Target disappeared from Prometheus target discovery.
|
||||
- alert: KubeVersionMismatch
|
||||
expr: count by(cluster) (count by(git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"}, "git_version", "$1", "git_version", "(v[0-9]*.[0-9]*).*"))) > 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: There are {{ $value }} different semantic versions of Kubernetes components running.
|
||||
summary: Different semantic versions of Kubernetes components running.
|
||||
- alert: KubeClientErrors
|
||||
expr: (sum by(cluster, instance, job, namespace) (rate(rest_client_requests_total{code=~"5.."}[5m])) / sum by(cluster, instance, job, namespace) (rate(rest_client_requests_total[5m]))) > 0.01
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
|
||||
summary: Kubernetes API server client is experiencing errors.
|
||||
- alert: KubeControllerManagerDown
|
||||
expr: absent(up{job="kube-controller-manager"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: KubeControllerManager has disappeared from Prometheus target discovery.
|
||||
summary: Target disappeared from Prometheus target discovery.
|
||||
- alert: KubeClientCertificateExpiration
|
||||
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by(job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.
|
||||
summary: Client certificate is about to expire.
|
||||
- alert: KubeClientCertificateExpiration
|
||||
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by(job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.
|
||||
summary: Client certificate is about to expire.
|
||||
- alert: KubeAggregatedAPIErrors
|
||||
expr: sum by(name, namespace, cluster) (increase(aggregator_unavailable_apiservice_total[10m])) > 4
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.
|
||||
summary: Kubernetes aggregated API has reported errors.
|
||||
- name: infra-alerts-03
|
||||
rules:
|
||||
- alert: KubeAggregatedAPIDown
|
||||
expr: (1 - max by(name, namespace, cluster) (avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
|
||||
summary: Kubernetes aggregated API is down.
|
||||
- alert: KubeAPIDown
|
||||
expr: absent(up{job="apiserver"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: KubeAPI has disappeared from Prometheus target discovery.
|
||||
summary: Target disappeared from Prometheus target discovery.
|
||||
- alert: KubeAPITerminatedRequests
|
||||
expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / (sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))) > 0.2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
|
||||
summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
|
||||
- alert: KubePersistentVolumeFillingUp
|
||||
expr: (kubelet_volume_stats_available_bytes{job="kubelet",namespace=~".*"} / kubelet_volume_stats_capacity_bytes{job="kubelet",namespace=~".*"}) < 0.03 and kubelet_volume_stats_used_bytes{job="kubelet",namespace=~".*"} > 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.
|
||||
summary: PersistentVolume is filling up.
|
||||
- alert: KubePersistentVolumeFillingUp
|
||||
expr: (kubelet_volume_stats_available_bytes{job="kubelet",namespace=~".*"} / kubelet_volume_stats_capacity_bytes{job="kubelet",namespace=~".*"}) < 0.15 and kubelet_volume_stats_used_bytes{job="kubelet",namespace=~".*"} > 0 and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet",namespace=~".*"}[6h], 4 * 24 * 3600) < 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days.
|
||||
summary: PersistentVolume is filling up.
|
||||
- alert: KubePersistentVolumeInodesFillingUp
|
||||
expr: (kubelet_volume_stats_inodes_free{job="kubelet",namespace=~".*"} / kubelet_volume_stats_inodes{job="kubelet",namespace=~".*"}) < 0.03 and kubelet_volume_stats_inodes_used{job="kubelet",namespace=~".*"} > 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.
|
||||
summary: PersistentVolumeInodes is filling up.
|
||||
- alert: KubePersistentVolumeInodesFillingUp
|
||||
expr: (kubelet_volume_stats_inodes_free{job="kubelet",namespace=~".*"} / kubelet_volume_stats_inodes{job="kubelet",namespace=~".*"}) < 0.15 and kubelet_volume_stats_inodes_used{job="kubelet",namespace=~".*"} > 0 and predict_linear(kubelet_volume_stats_inodes_free{job="kubelet",namespace=~".*"}[6h], 4 * 24 * 3600) < 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.
|
||||
summary: PersistentVolumeInodes are filling up.
|
||||
- alert: KubePersistentVolumeErrors
|
||||
expr: kube_persistentvolume_status_phase{job="kube-state-metrics",phase=~"Failed|Pending"} > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
|
||||
summary: PersistentVolume is having issues with provisioning.
|
||||
- alert: KubeCPUOvercommit
|
||||
expr: sum(namespace_cpu:kube_pod_container_resource_requests:sum) - (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0 and (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Cluster has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
|
||||
summary: Cluster has overcommitted CPU resource requests.
|
||||
- alert: KubeMemoryOvercommit
|
||||
expr: sum(namespace_memory:kube_pod_container_resource_requests:sum) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 and (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.
|
||||
summary: Cluster has overcommitted memory resource requests.
|
||||
- alert: KubeCPUQuotaOvercommit
|
||||
expr: sum(min without(resource) (kube_resourcequota{job="kube-state-metrics",resource=~"(cpu|requests.cpu)",type="hard"})) / sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) > 1.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Cluster has overcommitted CPU resource requests for Namespaces.
|
||||
summary: Cluster has overcommitted CPU resource requests.
|
||||
- alert: KubeMemoryQuotaOvercommit
|
||||
expr: sum(min without(resource) (kube_resourcequota{job="kube-state-metrics",resource=~"(memory|requests.memory)",type="hard"})) / sum(kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) > 1.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Cluster has overcommitted memory resource requests for Namespaces.
|
||||
summary: Cluster has overcommitted memory resource requests.
|
||||
- alert: KubeQuotaAlmostFull
|
||||
expr: kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring(instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) > 0.9 < 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
|
||||
summary: Namespace quota is going to be full.
|
||||
- alert: KubeQuotaFullyUsed
|
||||
expr: kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring(instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) == 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
|
||||
summary: Namespace quota is fully used.
|
||||
- alert: KubeQuotaExceeded
|
||||
expr: kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring(instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) > 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
|
||||
summary: Namespace quota has exceeded the limits.
|
||||
- alert: CPUThrottlingHigh
|
||||
expr: sum by(container, pod, namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) / sum by(container, pod, namespace) (increase(container_cpu_cfs_periods_total[5m])) > (25 / 100)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
description: The {{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.
|
||||
summary: Processes experience elevated CPU throttling.
|
||||
- alert: KubePodCrashLooping
|
||||
expr: max_over_time(kube_pod_container_status_waiting_reason{job="kube-state-metrics",namespace=~".*",reason="CrashLoopBackOff"}[5m]) >= 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason:"CrashLoopBackOff").
|
||||
summary: Pod is crash looping.
|
||||
- alert: KubePodNotReady
|
||||
expr: sum by(namespace, pod, cluster) (max by(namespace, pod, cluster) (kube_pod_status_phase{job="kube-state-metrics",namespace=~".*",phase=~"Pending|Unknown"}) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) has been in a non-ready state for longer than 15 minutes.
|
||||
summary: Pod has been in a non-ready state for more than 15 minutes.
|
||||
- alert: KubeDeploymentGenerationMismatch
|
||||
expr: kube_deployment_status_observed_generation{job="kube-state-metrics",namespace=~".*"} != kube_deployment_metadata_generation{job="kube-state-metrics",namespace=~".*"}
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.
|
||||
summary: Deployment generation mismatch due to possible roll-back
|
||||
- alert: KubeDeploymentReplicasMismatch
|
||||
expr: (kube_deployment_spec_replicas{job="kube-state-metrics",namespace=~".*"} > kube_deployment_status_replicas_available{job="kube-state-metrics",namespace=~".*"}) and (changes(kube_deployment_status_replicas_updated{job="kube-state-metrics",namespace=~".*"}[10m]) == 0)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.
|
||||
summary: Deployment has not matched the expected number of replicas.
|
||||
- name: infra-alerts-04
|
||||
rules:
|
||||
- alert: KubeStatefulSetReplicasMismatch
|
||||
expr: (kube_statefulset_status_replicas_ready{job="kube-state-metrics",namespace=~".*"} != kube_statefulset_status_replicas{job="kube-state-metrics",namespace=~".*"}) and (changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics",namespace=~".*"}[10m]) == 0)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.
|
||||
summary: Deployment has not matched the expected number of replicas.
|
||||
- alert: KubeStatefulSetGenerationMismatch
|
||||
expr: kube_statefulset_status_observed_generation{job="kube-state-metrics",namespace=~".*"} != kube_statefulset_metadata_generation{job="kube-state-metrics",namespace=~".*"}
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
|
||||
summary: StatefulSet generation mismatch due to possible roll-back
|
||||
- alert: KubeStatefulSetUpdateNotRolledOut
|
||||
expr: (max without(revision) (kube_statefulset_status_current_revision{job="kube-state-metrics",namespace=~".*"} unless kube_statefulset_status_update_revision{job="kube-state-metrics",namespace=~".*"}) * (kube_statefulset_replicas{job="kube-state-metrics",namespace=~".*"} != kube_statefulset_status_replicas_updated{job="kube-state-metrics",namespace=~".*"})) and (changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics",namespace=~".*"}[5m]) == 0)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
|
||||
summary: StatefulSet update has not been rolled out.
|
||||
- alert: KubeDaemonSetRolloutStuck
|
||||
expr: ((kube_daemonset_status_current_number_scheduled{job="kube-state-metrics",namespace=~".*"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics",namespace=~".*"}) or (kube_daemonset_status_number_misscheduled{job="kube-state-metrics",namespace=~".*"} != 0) or (kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics",namespace=~".*"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics",namespace=~".*"}) or (kube_daemonset_status_number_available{job="kube-state-metrics",namespace=~".*"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics",namespace=~".*"})) and (changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics",namespace=~".*"}[5m]) == 0)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.
|
||||
summary: DaemonSet rollout is stuck.
|
||||
- alert: KubeContainerWaiting
|
||||
expr: sum by(namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",namespace=~".*"}) > 0
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.
|
||||
summary: Pod container waiting longer than 1 hour
|
||||
- alert: KubeDaemonSetNotScheduled
|
||||
expr: kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics",namespace=~".*"} - kube_daemonset_status_current_number_scheduled{job="kube-state-metrics",namespace=~".*"} > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.
|
||||
summary: DaemonSet pods are not scheduled.
|
||||
- alert: KubeDaemonSetMisScheduled
|
||||
expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics",namespace=~".*"} > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.
|
||||
summary: DaemonSet pods are misscheduled.
|
||||
- alert: KubeJobNotCompleted
|
||||
expr: time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics",namespace=~".*"} and kube_job_status_active{job="kube-state-metrics",namespace=~".*"} > 0) > 43200
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.
|
||||
summary: Job did not complete in time
|
||||
- alert: KubeJobFailed
|
||||
expr: kube_job_failed{job="kube-state-metrics",namespace=~".*"} > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.
|
||||
summary: Job failed to complete.
|
||||
- alert: KubeHpaReplicasMismatch
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics",namespace=~".*"} != kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics",namespace=~".*"}) and (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics",namespace=~".*"} > kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics",namespace=~".*"}) and (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics",namespace=~".*"} < kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics",namespace=~".*"}) and changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics",namespace=~".*"}[15m]) == 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.
|
||||
summary: HPA has not matched descired number of replicas.
|
||||
- alert: KubeHpaMaxedOut
|
||||
expr: kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics",namespace=~".*"} == kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics",namespace=~".*"}
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.
|
||||
summary: HPA is running at max replicas
|
||||
- alert: KubeStateMetricsListErrors
|
||||
expr: (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) / sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m]))) > 0.01
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects or at all.
|
||||
summary: kube-state-metrics is experiencing errors in list operations.
|
||||
- alert: KubeStateMetricsWatchErrors
|
||||
expr: (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) / sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m]))) > 0.01
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects or at all.
|
||||
summary: kube-state-metrics is experiencing errors in watch operations.
|
||||
- alert: KubeStateMetricsShardingMismatch
|
||||
expr: stdvar(kube_state_metrics_total_shards{job="kube-state-metrics"}) != 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all.
|
||||
summary: kube-state-metrics sharding is misconfigured.
|
||||
- alert: KubeStateMetricsShardsMissing
|
||||
expr: 2 ^ max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1 - sum(2 ^ max by(shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"})) != 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed.
|
||||
summary: kube-state-metrics shards are missing.
|
||||
- alert: KubeAPIErrorBudgetBurn
|
||||
expr: sum(apiserver_request:burnrate1h) > (14.4 * 0.01) and sum(apiserver_request:burnrate5m) > (14.4 * 0.01)
|
||||
for: 2m
|
||||
labels:
|
||||
long: 1h
|
||||
severity: critical
|
||||
short: 5m
|
||||
annotations:
|
||||
description: The API server is burning too much error budget.
|
||||
summary: The API server is burning too much error budget.
|
||||
- alert: KubeAPIErrorBudgetBurn
|
||||
expr: sum(apiserver_request:burnrate6h) > (6 * 0.01) and sum(apiserver_request:burnrate30m) > (6 * 0.01)
|
||||
for: 15m
|
||||
labels:
|
||||
long: 6h
|
||||
severity: critical
|
||||
short: 30m
|
||||
annotations:
|
||||
description: The API server is burning too much error budget.
|
||||
summary: The API server is burning too much error budget.
|
||||
- alert: KubeAPIErrorBudgetBurn
|
||||
expr: sum(apiserver_request:burnrate1d) > (3 * 0.01) and sum(apiserver_request:burnrate2h) > (3 * 0.01)
|
||||
for: 1d
|
||||
labels:
|
||||
long: 1d
|
||||
severity: warning
|
||||
short: 2h
|
||||
annotations:
|
||||
description: The API server is burning too much error budget.
|
||||
summary: The API server is burning too much error budget.
|
||||
- alert: KubeAPIErrorBudgetBurn
|
||||
expr: sum(apiserver_request:burnrate3d) > (1 * 0.01) and sum(apiserver_request:burnrate6h) > (1 * 0.01)
|
||||
for: 3h
|
||||
labels:
|
||||
long: 3d
|
||||
severity: warning
|
||||
short: 6h
|
||||
annotations:
|
||||
description: The API server is burning too much error budget.
|
||||
summary: The API server is burning too much error budget.
|
||||
- alert: TargetDown
|
||||
expr: 100 * (count by(job, namespace, service) (up == 0) / count by(job, namespace, service) (up)) > 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: The {{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.
|
||||
- name: infra-alerts-05
|
||||
rules:
|
||||
- alert: Watchdog
|
||||
expr: vector(1)
|
||||
labels:
|
||||
severity: none
|
||||
annotations:
|
||||
description: This is an alert meant to ensure that the entire alerting pipeline is functional. This alert is always firing, therefore it should always be firing in Alertmanager and always fire against a receiver. There are integrations with various notification mechanisms that send a notification when this alert is not firing. For example the "DeadMansSnitch" integration in PagerDuty.
|
||||
- alert: InfoInhibitor
|
||||
expr: ALERTS{severity="info"} == 1 unless on(namespace) ALERTS{alertname!="InfoInhibitor",alertstate="firing",severity=~"warning|critical"} == 1
|
||||
labels:
|
||||
severity: none
|
||||
annotations:
|
||||
description: This is an alert that is used to inhibit info alerts. By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with other alerts. This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a severity of 'warning' or 'critical' starts firing on the same namespace. This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
|
||||
- alert: etcdInsufficientMembers
|
||||
expr: sum by(job) (up{job=~".*etcd.*"} == bool 1) < ((count by(job) (up{job=~".*etcd.*"}) + 1) / 2)
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":insufficient members ({{ $value }}).
|
||||
- alert: etcdHighNumberOfLeaderChanges
|
||||
expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":instance {{ $labels.instance }} has seen {{ $value }} leader changes within the last hour.
|
||||
- alert: etcdNoLeader
|
||||
expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: message:etcd cluster "{{ $labels.job }}":member {{ $labels.instance }} has no leader.
|
||||
- alert: etcdHighNumberOfFailedGRPCRequests
|
||||
expr: 100 * sum by(job, instance, grpc_service, grpc_method) (rate(grpc_server_handled_total{grpc_code!="OK",job=~".*etcd.*"}[5m])) / sum by(job, instance, grpc_service, grpc_method) (rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":{{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdGRPCRequestsSlow
|
||||
expr: histogram_quantile(0.99, sum by(job, instance, grpc_service, grpc_method, le) (rate(grpc_server_handling_seconds_bucket{grpc_type="unary",job=~".*etcd.*"}[5m]))) > 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdMemberCommunicationSlow
|
||||
expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: message:etcd cluster "{{ $labels.job }}":member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdHighNumberOfFailedProposals
|
||||
expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":{{ $value }} proposal failures within the last hour on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdHighFsyncDurations
|
||||
expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":99th percentile fync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdHighCommitDurations
|
||||
expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.25
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: etcd cluster "{{ $labels.job }}":99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdHighNumberOfFailedHTTPRequests
|
||||
expr: sum by(method) (rate(etcd_http_failed_total{code!="404",job=~".*etcd.*"}[5m])) / sum by(method) (rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) > 0.01
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: The {{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}
|
||||
- alert: etcdHighNumberOfFailedHTTPRequests
|
||||
expr: sum by(method) (rate(etcd_http_failed_total{code!="404",job=~".*etcd.*"}[5m])) / sum by(method) (rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: The {{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}.
|
||||
- alert: etcdHTTPRequestsSlow
|
||||
expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow.
|
||||
EOF
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
resource "grafana_dashboard" "workloads" {
|
||||
count = var.enable_dashboards ? 1 : 0
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/workloads.json")
|
||||
}
|
||||
|
||||
resource "grafana_dashboard" "nodes" {
|
||||
count = var.enable_dashboards ? 1 : 0
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/nodes.json")
|
||||
}
|
||||
|
||||
resource "grafana_dashboard" "nsworkload" {
|
||||
count = var.enable_dashboards ? 1 : 0
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/namespace-workloads.json")
|
||||
}
|
||||
|
||||
|
||||
resource "grafana_dashboard" "kubelet" {
|
||||
count = var.enable_dashboards ? 1 : 0
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/kubelet.json")
|
||||
}
|
||||
|
||||
resource "grafana_dashboard" "cluster" {
|
||||
count = var.enable_dashboards ? 1 : 0
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/cluster.json")
|
||||
}
|
||||
|
||||
resource "grafana_dashboard" "nodeexp_nodes" {
|
||||
count = var.enable_dashboards ? 1 : 0
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/nodeexporter-nodes.json")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
data "aws_partition" "current" {}
|
||||
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
data "aws_region" "current" {}
|
||||
|
||||
data "aws_eks_cluster" "eks_cluster" {
|
||||
name = var.eks_cluster_id
|
||||
}
|
||||
|
||||
locals {
|
||||
name = "adot-collector-kubeprometheus"
|
||||
namespace = try(var.helm_config.namespace, local.name)
|
||||
|
||||
eks_oidc_issuer_url = replace(data.aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer, "https://", "")
|
||||
eks_cluster_endpoint = data.aws_eks_cluster.eks_cluster.endpoint
|
||||
eks_cluster_version = data.aws_eks_cluster.eks_cluster.version
|
||||
|
||||
context = {
|
||||
aws_caller_identity_account_id = data.aws_caller_identity.current.account_id
|
||||
aws_caller_identity_arn = data.aws_caller_identity.current.arn
|
||||
aws_eks_cluster_endpoint = local.eks_cluster_endpoint
|
||||
aws_partition_id = data.aws_partition.current.partition
|
||||
aws_region_name = data.aws_region.current.name
|
||||
eks_cluster_id = var.eks_cluster_id
|
||||
eks_oidc_issuer_url = local.eks_oidc_issuer_url
|
||||
eks_oidc_provider_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.eks_oidc_issuer_url}"
|
||||
tags = var.tags
|
||||
irsa_iam_role_path = var.irsa_iam_role_path
|
||||
irsa_iam_permissions_boundary = var.irsa_iam_permissions_boundary
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
module "operator" {
|
||||
source = "./add-ons/adot-operator"
|
||||
count = var.enable_amazon_eks_adot ? 1 : 0
|
||||
|
||||
enable_cert_manager = var.enable_cert_manager
|
||||
kubernetes_version = local.eks_cluster_version
|
||||
addon_context = local.context
|
||||
}
|
||||
|
||||
resource "helm_release" "kube_state_metrics" {
|
||||
count = var.enable_kube_state_metrics ? 1 : 0
|
||||
chart = var.ksm_config.helm_chart_name
|
||||
create_namespace = var.ksm_config.create_namespace
|
||||
namespace = var.ksm_config.k8s_namespace
|
||||
name = var.ksm_config.helm_release_name
|
||||
version = var.ksm_config.helm_chart_version
|
||||
repository = var.ksm_config.helm_repo_url
|
||||
|
||||
dynamic "set" {
|
||||
for_each = var.ksm_config.helm_settings
|
||||
content {
|
||||
name = set.key
|
||||
value = set.value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "helm_release" "prometheus_node_exporter" {
|
||||
count = var.enable_node_exporter ? 1 : 0
|
||||
chart = var.ne_config.helm_chart_name
|
||||
create_namespace = var.ne_config.create_namespace
|
||||
namespace = var.ne_config.k8s_namespace
|
||||
name = var.ne_config.helm_release_name
|
||||
version = var.ne_config.helm_chart_version
|
||||
repository = var.ne_config.helm_repo_url
|
||||
|
||||
dynamic "set" {
|
||||
for_each = var.ne_config.helm_settings
|
||||
content {
|
||||
name = set.key
|
||||
value = set.value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module "helm_addon" {
|
||||
source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.13.1"
|
||||
|
||||
helm_config = merge(
|
||||
{
|
||||
name = local.name
|
||||
chart = "${path.module}/otel-config"
|
||||
version = "0.4.0"
|
||||
namespace = local.namespace
|
||||
description = "ADOT helm Chart deployment configuration"
|
||||
},
|
||||
var.helm_config
|
||||
)
|
||||
|
||||
set_values = [
|
||||
{
|
||||
name = "ampurl"
|
||||
value = "${var.managed_prometheus_workspace_endpoint}api/v1/remote_write"
|
||||
},
|
||||
{
|
||||
name = "region"
|
||||
value = var.managed_prometheus_workspace_region
|
||||
},
|
||||
{
|
||||
name = "ekscluster"
|
||||
value = local.context.eks_cluster_id
|
||||
},
|
||||
{
|
||||
name = "globalScrapeInterval"
|
||||
value = var.prometheus_config.global_scrape_interval
|
||||
},
|
||||
{
|
||||
name = "globalScrapeTimeout"
|
||||
value = var.prometheus_config.global_scrape_timeout
|
||||
},
|
||||
{
|
||||
name = "accountId"
|
||||
value = local.context.aws_caller_identity_account_id
|
||||
},
|
||||
{
|
||||
name = "enableTracing"
|
||||
value = var.enable_tracing
|
||||
},
|
||||
{
|
||||
name = "otlpHttpEndpoint"
|
||||
value = var.tracing_config.otlp_http_endpoint
|
||||
},
|
||||
{
|
||||
name = "otlpGrpcEndpoint"
|
||||
value = var.tracing_config.otlp_grpc_endpoint
|
||||
},
|
||||
{
|
||||
name = "tracingTimeout"
|
||||
value = var.tracing_config.timeout
|
||||
},
|
||||
{
|
||||
name = "tracingSendBatchSize"
|
||||
value = var.tracing_config.send_batch_size
|
||||
},
|
||||
{
|
||||
name = "enableCustomMetrics"
|
||||
value = var.enable_custom_metrics
|
||||
},
|
||||
{
|
||||
name = "customMetricsPorts"
|
||||
value = format(".*:(%s)$", join("|", var.custom_metrics_config.ports))
|
||||
},
|
||||
{
|
||||
name = "customMetricsDroppedSeriesPrefixes"
|
||||
value = format("(%s.*)$", join(".*|", var.custom_metrics_config.dropped_series_prefixes))
|
||||
},
|
||||
{
|
||||
name = "enable_java"
|
||||
value = var.enable_java
|
||||
},
|
||||
{
|
||||
name = "javaScrapeSampleLimit"
|
||||
value = var.java_config.scrape_sample_limit
|
||||
},
|
||||
{
|
||||
name = "enable_nginx"
|
||||
value = var.enable_nginx
|
||||
},
|
||||
{
|
||||
name = "nginxScrapeSampleLimit"
|
||||
value = var.nginx_config.scrape_sample_limit
|
||||
},
|
||||
{
|
||||
name = "nginxPrometheusMetricsEndpoint"
|
||||
value = var.nginx_config.prometheus_metrics_endpoint
|
||||
}
|
||||
]
|
||||
|
||||
irsa_config = {
|
||||
create_kubernetes_namespace = true
|
||||
kubernetes_namespace = local.namespace
|
||||
create_kubernetes_service_account = true
|
||||
kubernetes_service_account = try(var.helm_config.service_account, local.name)
|
||||
irsa_iam_policies = [
|
||||
"arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess",
|
||||
"arn:${data.aws_partition.current.partition}:iam::aws:policy/AWSXrayWriteOnlyAccess"
|
||||
]
|
||||
}
|
||||
|
||||
addon_context = local.context
|
||||
|
||||
depends_on = [module.operator]
|
||||
}
|
||||
|
||||
module "java_monitoring" {
|
||||
source = "./patterns/java"
|
||||
count = var.enable_java ? 1 : 0
|
||||
|
||||
managed_prometheus_workspace_id = var.managed_prometheus_workspace_id
|
||||
enable_alerting_rules = var.java_config.enable_alerting_rules
|
||||
dashboards_folder_id = var.dashboards_folder_id
|
||||
}
|
||||
|
||||
module "nginx_monitoring" {
|
||||
source = "./patterns/nginx"
|
||||
count = var.enable_nginx ? 1 : 0
|
||||
|
||||
managed_prometheus_workspace_id = var.managed_prometheus_workspace_id
|
||||
enable_alerting_rules = var.nginx_config.enable_alerting_rules
|
||||
dashboards_folder_id = var.dashboards_folder_id
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v2
|
||||
name: opentelemetry
|
||||
description: A Helm chart to install otel operator
|
||||
type: application
|
||||
version: 0.2.0
|
||||
appVersion: v0.1.0
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: otel-prometheus-role
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- extensions
|
||||
resources:
|
||||
- ingresses
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- nonResourceURLs:
|
||||
- /metrics
|
||||
verbs:
|
||||
- get
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: otel-prometheus-role-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: otel-prometheus-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: adot-collector-kubeprometheus
|
||||
namespace: adot-collector-kubeprometheus
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
ampurl: ${amp_url}
|
||||
region: ${region}
|
||||
ekscluster: ${eks_cluster}
|
||||
|
||||
globalScrapeTimeout: ${global_scrape_timeout}
|
||||
globalScrapeSampleLimit: ${global_scrape_sample_limit}
|
||||
|
||||
enableTracing: ${enable_tracing}
|
||||
otlpGrpcEndpoint: ${otlp_grpc_endpoint}
|
||||
otlpHttpEndpoint: ${otlp_http_endpoint}
|
||||
tracingTimeout: ${tracing_timeout}
|
||||
tracingSendBatchSize: ${tracing_send_batch_size}
|
||||
|
||||
enableCustomMetrics: ${enable_custom_metrics}
|
||||
customMetricsPorts: ${custom_metrics_ports}
|
||||
customMetricsDroppedSeriesPrefixes: ${custom_metrics_dropped_series_prefixes}
|
||||
|
||||
enableJava: ${enable_java}
|
||||
javaScrapeSampleLimit: ${java_scrape_sample_limit}
|
||||
|
||||
enableNginx: ${enable_nginx}
|
||||
nginxScrapeSampleLimit: ${nginx_scrape_sample_limit}
|
||||
nginxPrometheusMetricsEndpoint: ${nginx_prometheus_metrics_endpoint}
|
||||
@@ -0,0 +1,21 @@
|
||||
output "grafana_dashboard_urls" {
|
||||
value = [concat(
|
||||
grafana_dashboard.workloads[*].url,
|
||||
grafana_dashboard.nodes[*].url,
|
||||
grafana_dashboard.nsworkload[*].url,
|
||||
grafana_dashboard.kubelet[*].url,
|
||||
grafana_dashboard.cluster[*].url,
|
||||
flatten(module.java_monitoring[*].grafana_dashboard_urls),
|
||||
flatten(module.nginx_monitoring[*].grafana_dashboard_urls),
|
||||
)]
|
||||
description = "URLs for dashboards created"
|
||||
}
|
||||
output "eks_cluster_version" {
|
||||
description = "EKS Cluster version"
|
||||
value = data.aws_eks_cluster.eks_cluster.version
|
||||
}
|
||||
|
||||
output "eks_cluster_id" {
|
||||
description = "EKS Cluster Id"
|
||||
value = var.eks_cluster_id
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
# Java patterns module
|
||||
|
||||
Provides monitoring for Java based workloads with the following resources:
|
||||
|
||||
- AWS Managed Grafana Dashboard and data source
|
||||
- Alerts and recording rules with AWS Managed Service for Prometheus
|
||||
|
||||
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
## Requirements
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
|
||||
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
|
||||
| <a name="requirement_grafana"></a> [grafana](#requirement\_grafana) | >= 1.25.0 |
|
||||
| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.4.1 |
|
||||
| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14 |
|
||||
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
|
||||
| <a name="provider_grafana"></a> [grafana](#provider\_grafana) | >= 1.25.0 |
|
||||
|
||||
## Modules
|
||||
|
||||
No modules.
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|------|------|
|
||||
| [aws_prometheus_rule_group_namespace.alerting_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource |
|
||||
| [aws_prometheus_rule_group_namespace.recording_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource |
|
||||
| [grafana_dashboard.this](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|---------|:--------:|
|
||||
| <a name="input_dashboards_folder_id"></a> [dashboards\_folder\_id](#input\_dashboards\_folder\_id) | Grafana folder ID for automatic dashboards | `string` | n/a | yes |
|
||||
| <a name="input_enable_alerting_rules"></a> [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no |
|
||||
| <a name="input_managed_prometheus_workspace_id"></a> [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no |
|
||||
|
||||
## Outputs
|
||||
|
||||
| Name | Description |
|
||||
|------|-------------|
|
||||
| <a name="output_grafana_dashboard_urls"></a> [grafana\_dashboard\_urls](#output\_grafana\_dashboard\_urls) | URLs for dashboards created |
|
||||
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,36 @@
|
||||
resource "aws_prometheus_rule_group_namespace" "recording_rules" {
|
||||
name = "accelerator-java-rules"
|
||||
workspace_id = var.managed_prometheus_workspace_id
|
||||
data = <<EOF
|
||||
groups:
|
||||
- name: default-metric
|
||||
rules:
|
||||
- record: metric:recording_rule
|
||||
expr: avg(rate(container_cpu_usage_seconds_total[5m]))
|
||||
EOF
|
||||
}
|
||||
|
||||
resource "aws_prometheus_rule_group_namespace" "alerting_rules" {
|
||||
count = var.enable_alerting_rules ? 1 : 0
|
||||
|
||||
name = "accelerator-java-alerting"
|
||||
workspace_id = var.managed_prometheus_workspace_id
|
||||
data = <<EOF
|
||||
groups:
|
||||
- name: default-alert
|
||||
rules:
|
||||
- alert: metric:alerting_rule
|
||||
expr: jvm_memory_bytes_used{job="java", area="heap"} / jvm_memory_bytes_max * 100 > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "JVM heap warning"
|
||||
description: "JVM heap of instance `{{$labels.instance}}` from application `{{$labels.application}}` is above 80% for one minute. (current=`{{$value}}%`)"
|
||||
EOF
|
||||
}
|
||||
|
||||
resource "grafana_dashboard" "this" {
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/default.json")
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
output "grafana_dashboard_urls" {
|
||||
value = [concat(
|
||||
grafana_dashboard.this[*].url,
|
||||
)]
|
||||
description = "URLs for dashboards created"
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
variable "enable_alerting_rules" {
|
||||
description = "Enables or disables Managed Prometheus alerting rules"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "managed_prometheus_workspace_id" {
|
||||
description = "Amazon Managed Prometheus Workspace ID"
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "dashboards_folder_id" {
|
||||
description = "Grafana folder ID for automatic dashboards"
|
||||
type = string
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
terraform {
|
||||
required_version = ">= 1.1.0"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 4.0.0"
|
||||
}
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = ">= 2.10"
|
||||
}
|
||||
kubectl = {
|
||||
source = "gavinbunney/kubectl"
|
||||
version = ">= 1.14"
|
||||
}
|
||||
helm = {
|
||||
source = "hashicorp/helm"
|
||||
version = ">= 2.4.1"
|
||||
}
|
||||
grafana = {
|
||||
source = "grafana/grafana"
|
||||
version = ">= 1.25.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
# Observability Pattern for Nginx
|
||||
|
||||
This module provides an automated experience around Observability for Nginx workloads.
|
||||
It provides the following resources:
|
||||
|
||||
- AWS Distro For OpenTelemetry Operator and Collector
|
||||
- AWS Managed Grafana Dashboard and data source
|
||||
- Alerts and recording rules with AWS Managed Service for Prometheus
|
||||
|
||||
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
## Requirements
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
|
||||
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
|
||||
| <a name="requirement_grafana"></a> [grafana](#requirement\_grafana) | >= 1.25.0 |
|
||||
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
|
||||
| <a name="provider_grafana"></a> [grafana](#provider\_grafana) | >= 1.25.0 |
|
||||
|
||||
## Modules
|
||||
|
||||
No modules.
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|------|------|
|
||||
| [aws_prometheus_rule_group_namespace.alerting_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource |
|
||||
| [grafana_dashboard.workloads](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|---------|:--------:|
|
||||
| <a name="input_dashboards_folder_id"></a> [dashboards\_folder\_id](#input\_dashboards\_folder\_id) | Grafana folder ID for automatic dashboards | `string` | n/a | yes |
|
||||
| <a name="input_enable_alerting_rules"></a> [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no |
|
||||
| <a name="input_managed_prometheus_workspace_id"></a> [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no |
|
||||
|
||||
## Outputs
|
||||
|
||||
| Name | Description |
|
||||
|------|-------------|
|
||||
| <a name="output_grafana_dashboard_urls"></a> [grafana\_dashboard\_urls](#output\_grafana\_dashboard\_urls) | URLs for dashboards created |
|
||||
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,44 @@
|
||||
resource "aws_prometheus_rule_group_namespace" "alerting_rules" {
|
||||
count = var.enable_alerting_rules ? 1 : 0
|
||||
|
||||
name = "accelerator-nginx-alerting"
|
||||
workspace_id = var.managed_prometheus_workspace_id
|
||||
data = <<EOF
|
||||
groups:
|
||||
- name: Nginx-HTTP-4xx-error-rate
|
||||
rules:
|
||||
- alert: metric:alerting_rule
|
||||
expr: sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Nginx high HTTP 4xx error rate (instance {{ $labels.instance }})
|
||||
description: "Too many HTTP requests with status 4xx (> 5%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
- name: Nginx-HTTP-5xx-error-rate
|
||||
rules:
|
||||
- alert: metric:alerting_rule
|
||||
expr: sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Nginx high HTTP 5xx error rate (instance {{ $labels.instance }})
|
||||
description: "Too many HTTP requests with status 5xx (> 5%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
- name: Nginx-high-latency
|
||||
rules:
|
||||
- alert: metric:alerting_rule
|
||||
expr: histogram_quantile(0.99, sum(rate(nginx_http_request_duration_seconds_bucket[2m])) by (host, node)) > 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Nginx latency high (instance {{ $labels.instance }})
|
||||
description: "Nginx p99 latency is higher than 3 seconds\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
EOF
|
||||
}
|
||||
resource "grafana_dashboard" "workloads" {
|
||||
folder = var.dashboards_folder_id
|
||||
config_json = file("${path.module}/dashboards/nginx.json")
|
||||
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
output "grafana_dashboard_urls" {
|
||||
value = [concat(grafana_dashboard.workloads[*].url)]
|
||||
description = "URLs for dashboards created"
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
variable "managed_prometheus_workspace_id" {
|
||||
description = "Amazon Managed Prometheus Workspace ID"
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "dashboards_folder_id" {
|
||||
type = string
|
||||
description = "Grafana folder ID for automatic dashboards"
|
||||
}
|
||||
|
||||
variable "enable_alerting_rules" {
|
||||
type = bool
|
||||
default = true
|
||||
description = "Enables or disables Managed Prometheus alerting rules"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
terraform {
|
||||
required_version = ">= 1.1.0"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 4.0.0"
|
||||
}
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = ">= 2.10"
|
||||
}
|
||||
grafana = {
|
||||
source = "grafana/grafana"
|
||||
version = ">= 1.25.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
# Prioritize recording rules over alerting rules for limits (10)
|
||||
|
||||
################################################################################################################################################
|
||||
# Recording rules ##############################################################################################################################
|
||||
################################################################################################################################################
|
||||
|
||||
resource "aws_prometheus_rule_group_namespace" "recording_rules" {
|
||||
name = "accelerator-infra-rules"
|
||||
workspace_id = var.managed_prometheus_workspace_id
|
||||
data = <<EOF
|
||||
groups:
|
||||
- name: infra-rules-01
|
||||
rules:
|
||||
- record: "node_namespace_pod:kube_pod_info:"
|
||||
expr: topk by(cluster, namespace, pod) (1, max by(cluster, node, namespace, pod) (label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")))
|
||||
- record: node:node_num_cpu:sum
|
||||
expr: count by(cluster, node) (sum by(node, cpu) (node_cpu_seconds_total{job="node-exporter"} * on(namespace, pod) group_left(node) topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:)))
|
||||
- record: :node_memory_MemAvailable_bytes:sum
|
||||
expr: sum by(cluster) (node_memory_MemAvailable_bytes{job="node-exporter"} or (node_memory_Buffers_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Slab_bytes{job="node-exporter"}))
|
||||
- record: cluster:node_cpu:ratio_rate5m
|
||||
expr: sum by (cluster) (rate(node_cpu_seconds_total{job="node-exporter",mode!="idle",mode!="iowait",mode!="steal"}[5m])) / count by (cluster) (sum by(cluster, instance, cpu) (node_cpu_seconds_total{job="node-exporter"}))
|
||||
- record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.99, sum by(cluster, instance, le) (rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet"})
|
||||
labels:
|
||||
quantile: 0.99
|
||||
- record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.9, sum by(cluster, instance, le) (rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet"})
|
||||
labels:
|
||||
quantile: 0.9
|
||||
- record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.5, sum by(cluster, instance, le) (rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet"})
|
||||
labels:
|
||||
quantile: 0.5
|
||||
- record: instance:node_num_cpu:sum
|
||||
expr: count without(cpu, mode) (node_cpu_seconds_total{job="node-exporter",mode="idle"})
|
||||
- record: instance:node_cpu_utilisation:rate5m
|
||||
expr: 1 - avg without(cpu) (sum without(mode) (rate(node_cpu_seconds_total{job="node-exporter",mode=~"idle|iowait|steal"}[5m])))
|
||||
- record: instance:node_load1_per_cpu:ratio
|
||||
expr: (node_load1{job="node-exporter"} / instance:node_num_cpu:sum{job="node-exporter"})
|
||||
- record: instance:node_memory_utilisation:ratio
|
||||
expr: 1 - ((node_memory_MemAvailable_bytes{job="node-exporter"} or (node_memory_Buffers_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Slab_bytes{job="node-exporter"})) / node_memory_MemTotal_bytes{job="node-exporter"})
|
||||
- record: instance:node_vmstat_pgmajfault:rate5m
|
||||
expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m])
|
||||
- record: instance_device:node_disk_io_time_seconds:rate5m
|
||||
expr: rate(node_disk_io_time_seconds_total{device=~"mmcblk.p.+|.*nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+",job="node-exporter"}[5m])
|
||||
- record: instance_device:node_disk_io_time_weighted_seconds:rate5m
|
||||
expr: rate(node_disk_io_time_weighted_seconds_total{device=~"mmcblk.p.+|.*nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+",job="node-exporter"}[5m])
|
||||
- record: instance:node_network_receive_bytes_excluding_lo:rate5m
|
||||
expr: sum without(device) (rate(node_network_receive_bytes_total{device!="lo",job="node-exporter"}[5m]))
|
||||
- record: instance:node_network_transmit_bytes_excluding_lo:rate5m
|
||||
expr: sum without(device) (rate(node_network_transmit_bytes_total{device!="lo",job="node-exporter"}[5m]))
|
||||
- record: instance:node_network_receive_drop_excluding_lo:rate5m
|
||||
expr: sum without(device) (rate(node_network_receive_drop_total{device!="lo",job="node-exporter"}[5m]))
|
||||
- record: instance:node_network_transmit_drop_excluding_lo:rate5m
|
||||
expr: sum without(device) (rate(node_network_transmit_drop_total{device!="lo",job="node-exporter"}[5m]))
|
||||
- record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.99, sum without(instance, pod) (rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.99
|
||||
- record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.99, sum without(instance, pod) (rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.99
|
||||
- name: infra-rules-02
|
||||
rules:
|
||||
- record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.99, sum without(instance, pod) (rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.99
|
||||
- record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.9, sum without(instance, pod) (rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.9
|
||||
- record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.9, sum without(instance, pod) (rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.9
|
||||
- record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.9, sum without(instance, pod) (rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.9
|
||||
- record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.5, sum without(instance, pod) (rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.5
|
||||
- record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.5, sum without(instance, pod) (rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.5
|
||||
- record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.5, sum without(instance, pod) (rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])))
|
||||
labels:
|
||||
quantile: 0.5
|
||||
- record: instance:node_cpu:rate:sum
|
||||
expr: sum by(instance) (rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m]))
|
||||
- record: instance:node_network_receive_bytes:rate:sum
|
||||
expr: sum by(instance) (rate(node_network_receive_bytes_total[3m]))
|
||||
- record: instance:node_network_transmit_bytes:rate:sum
|
||||
expr: sum by(instance) (rate(node_network_transmit_bytes_total[3m]))
|
||||
- record: instance:node_cpu:ratio
|
||||
expr: sum without(cpu, mode) (rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) / on(instance) group_left() count by(instance) (sum by(instance, cpu) (node_cpu_seconds_total))
|
||||
- record: cluster:node_cpu:sum_rate5m
|
||||
expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
|
||||
- record: cluster:node_cpu:ratio
|
||||
expr: cluster:node_cpu:sum_rate5m / count(sum by(instance, cpu) (node_cpu_seconds_total))
|
||||
- record: count:up1
|
||||
expr: count without(instance, pod, node) (up == 1)
|
||||
- record: count:up0
|
||||
expr: count without(instance, pod, node) (up == 0)
|
||||
- record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.99, sum by(cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[5m]))) > 0
|
||||
labels:
|
||||
quantile: 0.99
|
||||
verb: read
|
||||
- record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile
|
||||
expr: histogram_quantile(0.99, sum by(cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0
|
||||
labels:
|
||||
quantile: 0.99
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate1d
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[1d]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate1h
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1h])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1h])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1h])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1h])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[1h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate2h
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[2h])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[2h])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[2h])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[2h])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[2h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h]))
|
||||
labels:
|
||||
verb: read
|
||||
- name: infra-rules-03
|
||||
rules:
|
||||
- record: apiserver_request:burnrate30m
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[30m])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[30m])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[30m])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[30m])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[30m]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate3d
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[3d])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[3d])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[3d])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[3d])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[3d]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate5m
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[5m])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[5m])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[5m])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[5m])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[5m]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate6h
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[6h])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[6h])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[6h])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[6h])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[6h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate1d
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[1d])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[1d]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
|
||||
labels:
|
||||
verb: read
|
||||
- record: apiserver_request:burnrate1d
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])) - ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",scope=~"resource|",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])) or vector(0)) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="5",scope="namespace",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])) + sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="30",scope="cluster",subresource!~"proxy|attach|log|exec|portforward",verb=~"LIST|GET"}[1d])))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"LIST|GET"}[1d]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate1h
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[1h])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[1h]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate2h
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[2h])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[2h]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate30m
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[30m])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[30m]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate3d
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[3d])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[3d]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate5m
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[5m])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: apiserver_request:burnrate6h
|
||||
expr: ((sum by(cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[6h])) - sum by(cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",le="1",subresource!~"proxy|attach|log|exec|portforward",verb=~"POST|PUT|PATCH|DELETE"}[6h]))) + sum by(cluster) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))) / sum by(cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
|
||||
labels:
|
||||
verb: write
|
||||
- record: code_verb:apiserver_request_total:increase30d
|
||||
expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30
|
||||
- record: code:apiserver_request_total:increase30d
|
||||
expr: sum by(cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"})
|
||||
labels:
|
||||
verb: read
|
||||
- record: code:apiserver_request_total:increase30d
|
||||
expr: sum by(cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
|
||||
labels:
|
||||
verb: write
|
||||
- record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h
|
||||
expr: sum by(cluster, verb, scope) (increase(apiserver_request_slo_duration_seconds_count[1h]))
|
||||
- record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d
|
||||
expr: sum by(cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30)
|
||||
- record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
|
||||
expr: sum by(cluster, namespace, pod, container) (irate(container_cpu_usage_seconds_total{image!="",job="kubelet"}[5m])) * on(cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}))
|
||||
- record: node_namespace_pod_container:container_memory_working_set_bytes
|
||||
expr: container_memory_working_set_bytes{image!="",job="kubelet"} * on(namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))
|
||||
- record: node_namespace_pod_container:container_memory_rss
|
||||
expr: container_memory_rss{image!="",job="kubelet"} * on(namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))
|
||||
- name: infra-rules-04
|
||||
rules:
|
||||
- record: node_namespace_pod_container:container_memory_cache
|
||||
expr: container_memory_cache{image!="",job="kubelet"} * on(namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))
|
||||
- record: node_namespace_pod_container:container_memory_swap
|
||||
expr: container_memory_swap{image!="",job="kubelet"} * on(namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))
|
||||
- record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests
|
||||
expr: kube_pod_container_resource_requests{job="kube-state-metrics",resource="memory"} * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))
|
||||
- record: namespace_memory:kube_pod_container_resource_requests:sum
|
||||
expr: sum by(namespace, cluster) (sum by(namespace, pod, cluster) (max by(namespace, pod, container, cluster) (kube_pod_container_resource_requests{job="kube-state-metrics",resource="memory"}) * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))
|
||||
- record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests
|
||||
expr: kube_pod_container_resource_requests{job="kube-state-metrics",resource="cpu"} * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))
|
||||
- record: namespace_cpu:kube_pod_container_resource_requests:sum
|
||||
expr: sum by(namespace, cluster) (sum by(namespace, pod, cluster) (max by(namespace, pod, container, cluster) (kube_pod_container_resource_requests{job="kube-state-metrics",resource="cpu"}) * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))
|
||||
- record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits
|
||||
expr: kube_pod_container_resource_limits{job="kube-state-metrics",resource="memory"} * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))
|
||||
- record: namespace_memory:kube_pod_container_resource_limits:sum
|
||||
expr: sum by(namespace, cluster) (sum by(namespace, pod, cluster) (max by(namespace, pod, container, cluster) (kube_pod_container_resource_limits{job="kube-state-metrics",resource="memory"}) * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))
|
||||
- record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits
|
||||
expr: kube_pod_container_resource_limits{job="kube-state-metrics",resource="cpu"} * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))
|
||||
- record: namespace_cpu:kube_pod_container_resource_limits:sum
|
||||
expr: sum by(namespace, cluster) (sum by(namespace, pod, cluster) (max by(namespace, pod, container, cluster) (kube_pod_container_resource_limits{job="kube-state-metrics",resource="cpu"}) * on(namespace, pod, cluster) group_left() max by(namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))
|
||||
- record: namespace_workload_pod:kube_pod_owner:relabel
|
||||
expr: max by(cluster, namespace, workload, pod) (label_replace(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="ReplicaSet"}, "replicaset", "$1", "owner_name", "(.*)") * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (1, max by(replicaset, namespace, owner_name) (kube_replicaset_owner{job="kube-state-metrics"})), "workload", "$1", "owner_name", "(.*)"))
|
||||
labels:
|
||||
workload_type: deployment
|
||||
- record: namespace_workload_pod:kube_pod_owner:relabel
|
||||
expr: max by(cluster, namespace, workload, pod) (label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="DaemonSet"}, "workload", "$1", "owner_name", "(.*)"))
|
||||
labels:
|
||||
workload_type: daemonset
|
||||
- record: namespace_workload_pod:kube_pod_owner:relabel
|
||||
expr: max by(cluster, namespace, workload, pod) (label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="StatefulSet"}, "workload", "$1", "owner_name", "(.*)"))
|
||||
labels:
|
||||
workload_type: statefulset
|
||||
- record: namespace_workload_pod:kube_pod_owner:relabel
|
||||
expr: max by(cluster, namespace, workload, pod) (label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="Job"}, "workload", "$1", "owner_name", "(.*)"))
|
||||
labels:
|
||||
workload_type: job
|
||||
EOF
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
variable "eks_cluster_id" {
|
||||
description = "EKS Cluster Id"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "enable_amazon_eks_adot" {
|
||||
description = "Enables the ADOT Operator on the EKS Cluster"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "enable_cert_manager" {
|
||||
description = "Allow reusing an existing installation of cert-manager"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "helm_config" {
|
||||
description = "Helm Config for Prometheus"
|
||||
type = any
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "irsa_iam_role_path" {
|
||||
description = "IAM role path for IRSA roles"
|
||||
type = string
|
||||
default = "/"
|
||||
}
|
||||
|
||||
variable "irsa_iam_permissions_boundary" {
|
||||
description = "IAM permissions boundary for IRSA roles"
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "managed_prometheus_workspace_endpoint" {
|
||||
description = "Amazon Managed Prometheus Workspace Endpoint"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "managed_prometheus_workspace_id" {
|
||||
description = "Amazon Managed Prometheus Workspace ID"
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "managed_prometheus_workspace_region" {
|
||||
description = "Amazon Managed Prometheus Workspace's Region"
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "dashboards_folder_id" {
|
||||
description = "Grafana folder ID for automatic dashboards"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "enable_alerting_rules" {
|
||||
description = "Enables or disables Managed Prometheus alerting rules"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "enable_dashboards" {
|
||||
description = "Enables or disables curated dashboards"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "enable_kube_state_metrics" {
|
||||
description = "Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "ksm_config" {
|
||||
description = "Kube State metrics configuration"
|
||||
type = object({
|
||||
create_namespace = bool
|
||||
k8s_namespace = string
|
||||
helm_chart_name = string
|
||||
helm_chart_version = string
|
||||
helm_release_name = string
|
||||
helm_repo_url = string
|
||||
helm_settings = map(string)
|
||||
helm_values = map(any)
|
||||
|
||||
scrape_interval = string
|
||||
scrape_timeout = string
|
||||
})
|
||||
|
||||
default = {
|
||||
create_namespace = true
|
||||
helm_chart_name = "kube-state-metrics"
|
||||
helm_chart_version = "4.24.0"
|
||||
helm_release_name = "kube-state-metrics"
|
||||
helm_repo_url = "https://prometheus-community.github.io/helm-charts"
|
||||
helm_settings = {}
|
||||
helm_values = {}
|
||||
k8s_namespace = "kube-system"
|
||||
|
||||
scrape_interval = "60s"
|
||||
scrape_timeout = "15s"
|
||||
}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "enable_node_exporter" {
|
||||
description = "Enables or disables Node exporter. Disabling this might affect some data in the dashboards"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "ne_config" {
|
||||
description = "Node exporter configuration"
|
||||
type = object({
|
||||
create_namespace = bool
|
||||
k8s_namespace = string
|
||||
helm_chart_name = string
|
||||
helm_chart_version = string
|
||||
helm_release_name = string
|
||||
helm_repo_url = string
|
||||
helm_settings = map(string)
|
||||
helm_values = map(any)
|
||||
|
||||
scrape_interval = string
|
||||
scrape_timeout = string
|
||||
})
|
||||
|
||||
default = {
|
||||
create_namespace = true
|
||||
helm_chart_name = "prometheus-node-exporter"
|
||||
helm_chart_version = "2.0.3"
|
||||
helm_release_name = "prometheus-node-exporter"
|
||||
helm_repo_url = "https://prometheus-community.github.io/helm-charts"
|
||||
helm_settings = {}
|
||||
helm_values = {}
|
||||
k8s_namespace = "prometheus-node-exporter"
|
||||
|
||||
scrape_interval = "60s"
|
||||
scrape_timeout = "60s"
|
||||
}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "prometheus_config" {
|
||||
description = "Controls default values such as scrape interval, timeouts and ports globally"
|
||||
type = object({
|
||||
global_scrape_interval = string
|
||||
global_scrape_timeout = string
|
||||
})
|
||||
|
||||
default = {
|
||||
global_scrape_interval = "60s"
|
||||
global_scrape_timeout = "15s"
|
||||
}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "enable_tracing" {
|
||||
description = "(Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "tracing_config" {
|
||||
description = "Configuration object for traces collection to AWS X-Ray"
|
||||
type = object({
|
||||
otlp_grpc_endpoint = string
|
||||
otlp_http_endpoint = string
|
||||
send_batch_size = number
|
||||
timeout = string
|
||||
})
|
||||
|
||||
default = {
|
||||
otlp_grpc_endpoint = "0.0.0.0:4317"
|
||||
otlp_http_endpoint = "0.0.0.0:4318"
|
||||
send_batch_size = 50
|
||||
timeout = "30s"
|
||||
}
|
||||
}
|
||||
|
||||
variable "enable_custom_metrics" {
|
||||
description = "Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "custom_metrics_config" {
|
||||
description = "Configuration object to enable custom metrics collection"
|
||||
type = object({
|
||||
ports = list(number)
|
||||
# paths = optional(list(string), ["/metrics"])
|
||||
# list of samples to be dropped by label prefix, ex: go_ -> discards go_.*
|
||||
dropped_series_prefixes = list(string)
|
||||
})
|
||||
|
||||
default = {
|
||||
ports = []
|
||||
dropped_series_prefixes = ["unspecified"]
|
||||
}
|
||||
}
|
||||
|
||||
variable "enable_java" {
|
||||
description = "Enable Java workloads monitoring, alerting and default dashboards"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "java_config" {
|
||||
description = "Configuration object for Java/JMX monitoring"
|
||||
type = object({
|
||||
enable_alerting_rules = bool
|
||||
scrape_sample_limit = number
|
||||
})
|
||||
|
||||
default = {
|
||||
enable_alerting_rules = true
|
||||
scrape_sample_limit = 1000
|
||||
}
|
||||
}
|
||||
|
||||
variable "enable_nginx" {
|
||||
description = "Enable NGINX workloads monitoring, alerting and default dashboards"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "nginx_config" {
|
||||
description = "Configuration object for NGINX monitoring"
|
||||
type = object({
|
||||
enable_alerting_rules = bool
|
||||
scrape_sample_limit = number
|
||||
prometheus_metrics_endpoint = string
|
||||
})
|
||||
|
||||
default = {
|
||||
enable_alerting_rules = true
|
||||
scrape_sample_limit = 1000
|
||||
prometheus_metrics_endpoint = "metrics"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
terraform {
|
||||
required_version = ">= 1.1.0"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 4.0.0"
|
||||
}
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = ">= 2.10"
|
||||
}
|
||||
kubectl = {
|
||||
source = "gavinbunney/kubectl"
|
||||
version = ">= 1.14"
|
||||
}
|
||||
helm = {
|
||||
source = "hashicorp/helm"
|
||||
version = ">= 2.4.1"
|
||||
}
|
||||
grafana = {
|
||||
source = "grafana/grafana"
|
||||
version = ">= 1.25.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user