From 5b52c7b8e16b74d624f0027119361e6c7495f751 Mon Sep 17 00:00:00 2001 From: Ela AWS <51791117+elamaran11@users.noreply.github.com> Date: Fri, 14 Apr 2023 04:20:24 -0400 Subject: [PATCH] GitOps support with Grafana Operator, External Secrets and FluxCD (#151) * Adding FluxCD and Grafana Operator Addons * Adding Helm release for Grafana Operator * Adding External Secrets * removing spot options * Fixing Documentation --------- --- docs/index.md | 3 + examples/existing-cluster-java/main.tf | 6 + examples/existing-cluster-nginx/main.tf | 6 + .../main.tf | 6 + modules/eks-monitoring/README.md | 16 ++- .../add-ons/external-secrets/README.md | 55 +++++++++ .../add-ons/external-secrets/locals.tf | 6 + .../add-ons/external-secrets/main.tf | 109 ++++++++++++++++++ .../add-ons/external-secrets/outputs.tf | 0 .../add-ons/external-secrets/variables.tf | 43 +++++++ .../add-ons/external-secrets/versions.tf | 18 +++ modules/eks-monitoring/dashboards.tf | 1 - modules/eks-monitoring/main.tf | 41 +++++++ modules/eks-monitoring/variables.tf | 83 +++++++++++++ 14 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 modules/eks-monitoring/add-ons/external-secrets/README.md create mode 100644 modules/eks-monitoring/add-ons/external-secrets/locals.tf create mode 100644 modules/eks-monitoring/add-ons/external-secrets/main.tf create mode 100644 modules/eks-monitoring/add-ons/external-secrets/outputs.tf create mode 100644 modules/eks-monitoring/add-ons/external-secrets/variables.tf create mode 100644 modules/eks-monitoring/add-ons/external-secrets/versions.tf diff --git a/docs/index.md b/docs/index.md index dab1b01..193cf4b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,6 +25,9 @@ traces collection, dashboards and alerts for monitoring: - NGINX workloads (running on Amazon EKS) - Java/JMX workloads (running on Amazon EKS) - Amazon Managed Service for Prometheus workspaces with Amazon CloudWatch +- Installs Grafana Operator to add AWS data sources and create Grafana Dashboards to Amazon Managed Grafana. +- Installs FluxCD to perform GitOps sync of a Git Repo to EKS Cluster. We will use this later for creating Grafana Dashboards and AWS datasources to Amazon Managed Grafana. +- Installs External Secrets Operator to retrieve and Sync the Grafana API keys. These modules can be directly configured in your existing Terraform configurations or ready to be deployed in our packaged diff --git a/examples/existing-cluster-java/main.tf b/examples/existing-cluster-java/main.tf index 6546f2b..832e84b 100644 --- a/examples/existing-cluster-java/main.tf +++ b/examples/existing-cluster-java/main.tf @@ -69,6 +69,12 @@ module "eks_monitoring" { # enable java metrics collection, dashboards and alerts rules creation enable_java = true + # deploys external-secrets in to the cluster + enable_external_secrets = true + grafana_api_key = var.grafana_api_key + target_secret_name = "grafana-admin-credentials" + target_secret_namespace = "grafana-operator" + eks_cluster_id = var.eks_cluster_id # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true' diff --git a/examples/existing-cluster-nginx/main.tf b/examples/existing-cluster-nginx/main.tf index 8249b82..507335e 100644 --- a/examples/existing-cluster-nginx/main.tf +++ b/examples/existing-cluster-nginx/main.tf @@ -67,6 +67,12 @@ module "eks_monitoring" { eks_cluster_id = var.eks_cluster_id + # deploys external-secrets in to the cluster + enable_external_secrets = true + grafana_api_key = var.grafana_api_key + target_secret_name = "grafana-admin-credentials" + target_secret_namespace = "grafana-operator" + # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true' # the intention to publish is overruled depending upon whether grafana dashboard folder is created by the observability accelerator enable_dashboards = module.aws_observability_accelerator.grafana_dashboard_folder_created ? var.enable_dashboards : false diff --git a/examples/existing-cluster-with-base-and-infra/main.tf b/examples/existing-cluster-with-base-and-infra/main.tf index ef0c7c9..3a9a580 100644 --- a/examples/existing-cluster-with-base-and-infra/main.tf +++ b/examples/existing-cluster-with-base-and-infra/main.tf @@ -77,6 +77,12 @@ module "eks_monitoring" { # reusing existing certificate manager? defaults to true enable_cert_manager = true + # deploys external-secrets in to the cluster + enable_external_secrets = true + grafana_api_key = var.grafana_api_key + target_secret_name = "grafana-admin-credentials" + target_secret_namespace = "grafana-operator" + # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true' # the intention to publish is overruled depending upon whether grafana dashboard folder is created by the observability accelerator enable_dashboards = module.aws_observability_accelerator.grafana_dashboard_folder_created ? var.enable_dashboards : false diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 319a293..781825f 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -4,7 +4,10 @@ This module provides EKS cluster monitoring with the following resources: - AWS Distro For OpenTelemetry Operator and Collector for Metrics and Traces - Logs with [AWS for FluentBit](https://github.com/aws/aws-for-fluent-bit) -- AWS Managed Grafana Dashboard and data source +- Installs Grafana Operator to add AWS data sources and create Grafana Dashboards to Amazon Managed Grafana. +- Installs FluxCD to perform GitOps sync of a Git Repo to EKS Cluster. We will use this later for creating Grafana Dashboards and AWS datasources to Amazon Managed Grafana. +- Installs External Secrets Operator to retrieve and Sync the Grafana API keys. +- Amazon Managed Grafana Dashboard and data source - Alerts and recording rules with AWS Managed Service for Prometheus This module makes use of the open source [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) @@ -33,6 +36,7 @@ This module makes use of the open source [kube-prometheus-stack](https://github. | Name | Source | Version | |------|--------|---------| +| [external\_secrets](#module\_external\_secrets) | ./add-ons/external-secrets | n/a | | [fluentbit\_logs](#module\_fluentbit\_logs) | ./add-ons/aws-for-fluentbit | n/a | | [helm\_addon](#module\_helm\_addon) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.26.0 | | [java\_monitoring](#module\_java\_monitoring) | ./patterns/java | n/a | @@ -51,6 +55,8 @@ This module makes use of the open source [kube-prometheus-stack](https://github. | [grafana_dashboard.nodes](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource | | [grafana_dashboard.nsworkload](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource | | [grafana_dashboard.workloads](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/dashboard) | resource | +| [helm_release.fluxcd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.grafana_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.prometheus_node_exporter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | @@ -70,6 +76,9 @@ This module makes use of the open source [kube-prometheus-stack](https://github. | [enable\_cert\_manager](#input\_enable\_cert\_manager) | Allow reusing an existing installation of cert-manager | `bool` | `true` | no | | [enable\_custom\_metrics](#input\_enable\_custom\_metrics) | Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included | `bool` | `false` | no | | [enable\_dashboards](#input\_enable\_dashboards) | Enables or disables curated dashboards | `bool` | `true` | no | +| [enable\_external\_secrets](#input\_enable\_external\_secrets) | Installs External Secrets to EKS Cluster | `bool` | `true` | no | +| [enable\_fluxcd](#input\_enable\_fluxcd) | Enables or disables FluxCD. Disabling this might affect some data in the dashboards | `bool` | `true` | no | +| [enable\_grafana\_operator](#input\_enable\_grafana\_operator) | Deploys Grafana Operator to EKS Cluster | `bool` | `true` | no | | [enable\_java](#input\_enable\_java) | Enable Java workloads monitoring, alerting and default dashboards | `bool` | `false` | no | | [enable\_kube\_state\_metrics](#input\_enable\_kube\_state\_metrics) | Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no | | [enable\_logs](#input\_enable\_logs) | Using AWS For FluentBit to collect cluster and application logs to Amazon CloudWatch | `bool` | `true` | no | @@ -77,6 +86,9 @@ This module makes use of the open source [kube-prometheus-stack](https://github. | [enable\_node\_exporter](#input\_enable\_node\_exporter) | Enables or disables Node exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no | | [enable\_recording\_rules](#input\_enable\_recording\_rules) | Enables or disables Managed Prometheus recording rules | `bool` | `true` | no | | [enable\_tracing](#input\_enable\_tracing) | (Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0 | `bool` | `false` | no | +| [flux\_config](#input\_flux\_config) | FluxCD configuration |
object({
create_namespace = bool
k8s_namespace = string
helm_chart_name = string
helm_chart_version = string
helm_release_name = string
helm_repo_url = string
helm_settings = map(string)
helm_values = map(any)
}) | {
"create_namespace": true,
"helm_chart_name": "flux2",
"helm_chart_version": "2.7.0",
"helm_release_name": "observability-fluxcd-addon",
"helm_repo_url": "https://fluxcd-community.github.io/helm-charts",
"helm_settings": {},
"helm_values": {},
"k8s_namespace": "flux-system"
} | no |
+| [go\_config](#input\_go\_config) | Grafana Operator configuration | object({
create_namespace = bool
helm_chart = string
helm_name = string
k8s_namespace = string
helm_release_name = string
helm_chart_version = string
}) | {
"create_namespace": true,
"helm_chart": "oci://ghcr.io/grafana-operator/helm-charts/grafana-operator",
"helm_chart_version": "v5.0.0-rc1",
"helm_name": "grafana-operator",
"helm_release_name": "grafana-operator",
"k8s_namespace": "grafana-operator"
} | no |
+| [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace | `string` | n/a | yes |
| [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no |
| [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `null` | no |
| [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no |
@@ -90,6 +102,8 @@ This module makes use of the open source [kube-prometheus-stack](https://github.
| [nginx\_config](#input\_nginx\_config) | Configuration object for NGINX monitoring | object({
enable_alerting_rules = bool
scrape_sample_limit = number
prometheus_metrics_endpoint = string
}) | {
"enable_alerting_rules": true,
"prometheus_metrics_endpoint": "metrics",
"scrape_sample_limit": 1000
} | no |
| [prometheus\_config](#input\_prometheus\_config) | Controls default values such as scrape interval, timeouts and ports globally | object({
global_scrape_interval = string
global_scrape_timeout = string
}) | {
"global_scrape_interval": "60s",
"global_scrape_timeout": "15s"
} | no |
| [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no |
+| [target\_secret\_name](#input\_target\_secret\_name) | Target secret in Kubernetes to store the Grafana API Key Secret | `string` | `"grafana-admin-credentials"` | no |
+| [target\_secret\_namespace](#input\_target\_secret\_namespace) | Target namespace of secret in Kubernetes to store the Grafana API Key Secret | `string` | `"grafana-operator"` | no |
| [tracing\_config](#input\_tracing\_config) | Configuration object for traces collection to AWS X-Ray | object({
otlp_grpc_endpoint = string
otlp_http_endpoint = string
send_batch_size = number
timeout = string
}) | {
"otlp_grpc_endpoint": "0.0.0.0:4317",
"otlp_http_endpoint": "0.0.0.0:4318",
"send_batch_size": 50,
"timeout": "30s"
} | no |
## Outputs
diff --git a/modules/eks-monitoring/add-ons/external-secrets/README.md b/modules/eks-monitoring/add-ons/external-secrets/README.md
new file mode 100644
index 0000000..248c2e5
--- /dev/null
+++ b/modules/eks-monitoring/add-ons/external-secrets/README.md
@@ -0,0 +1,55 @@
+# External Secrets Operator Kubernetes addon
+
+This deploys an EKS Cluster with the External Secrets Operator. The cluster is populated with a ClusterSecretStore and ExternalSecret using Grafana API Key secret from AWS Secret Manager. A secret store for each AWS Secret Manager is created. Store use IRSA (IAM Roles For Service Account) to retrieve the secret values from AWS.
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | >= 1.0.0 |
+| [aws](#requirement\_aws) | >= 3.72 |
+| [kubectl](#requirement\_kubectl) | >= 1.14 |
+| [kubernetes](#requirement\_kubernetes) | >= 2.10 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [aws](#provider\_aws) | >= 3.72 |
+| [kubectl](#provider\_kubectl) | >= 1.14 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| [cluster\_secretstore\_role](#module\_cluster\_secretstore\_role) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa | v4.28.0 |
+| [external\_secrets](#module\_external\_secrets) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/external-secrets | v4.28.0 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_policy.cluster_secretstore](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_kms_key.secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
+| [aws_secretsmanager_secret.secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource |
+| [aws_secretsmanager_secret_version.secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource |
+| [kubectl_manifest.cluster_secretstore](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.secret](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [addon\_context](#input\_addon\_context) | Input configuration for the addon | object({
aws_caller_identity_account_id = string
aws_caller_identity_arn = string
aws_eks_cluster_endpoint = string
aws_partition_id = string
aws_region_name = string
eks_cluster_id = string
eks_oidc_issuer_url = string
eks_oidc_provider_arn = string
irsa_iam_role_path = string
irsa_iam_permissions_boundary = string
tags = map(string)
}) | n/a | yes |
+| [enable\_external\_secrets](#input\_enable\_external\_secrets) | Enable external-secrets | `bool` | `true` | no |
+| [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace | `string` | n/a | yes |
+| [helm\_config](#input\_helm\_config) | Helm provider config for external secrets | `any` | `{}` | no |
+| [target\_secret\_name](#input\_target\_secret\_name) | Name to store the secret for Grafana API Key | `string` | n/a | yes |
+| [target\_secret\_namespace](#input\_target\_secret\_namespace) | Namespace to store the secret for Grafana API Key | `string` | n/a | yes |
+
+## Outputs
+
+No outputs.
+
diff --git a/modules/eks-monitoring/add-ons/external-secrets/locals.tf b/modules/eks-monitoring/add-ons/external-secrets/locals.tf
new file mode 100644
index 0000000..29d3fc9
--- /dev/null
+++ b/modules/eks-monitoring/add-ons/external-secrets/locals.tf
@@ -0,0 +1,6 @@
+locals {
+ name = "external-secrets"
+ namespace = "external-secrets"
+ cluster_secretstore_name = "cluster-secretstore-sm"
+ cluster_secretstore_sa = "cluster-secretstore-sa"
+}
diff --git a/modules/eks-monitoring/add-ons/external-secrets/main.tf b/modules/eks-monitoring/add-ons/external-secrets/main.tf
new file mode 100644
index 0000000..ef6ecae
--- /dev/null
+++ b/modules/eks-monitoring/add-ons/external-secrets/main.tf
@@ -0,0 +1,109 @@
+module "external_secrets" {
+ source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/external-secrets?ref=v4.28.0"
+ count = var.enable_external_secrets ? 1 : 0
+
+ helm_config = var.helm_config
+ addon_context = var.addon_context
+}
+
+data "aws_region" "current" {}
+
+#---------------------------------------------------------------
+# External Secrets Operator - Secret
+#---------------------------------------------------------------
+
+resource "aws_kms_key" "secrets" {
+ enable_key_rotation = true
+}
+
+module "cluster_secretstore_role" {
+ source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa?ref=v4.28.0"
+ kubernetes_namespace = local.namespace
+ create_kubernetes_namespace = false
+ kubernetes_service_account = local.cluster_secretstore_sa
+ irsa_iam_policies = [aws_iam_policy.cluster_secretstore.arn]
+ eks_cluster_id = var.addon_context.eks_cluster_id
+ eks_oidc_provider_arn = var.addon_context.eks_oidc_provider_arn
+ depends_on = [module.external_secrets]
+}
+
+resource "aws_iam_policy" "cluster_secretstore" {
+ name_prefix = local.cluster_secretstore_sa
+ policy = <