diff --git a/docs/eks/multiaccount.md b/docs/eks/multiaccount.md new file mode 100644 index 0000000..934fa15 --- /dev/null +++ b/docs/eks/multiaccount.md @@ -0,0 +1,119 @@ +# AWS EKS Cross Account Observability + +This example shows how to use the [AWS Observability Accelerator](https://github.com/aws-observability/terraform-aws-observability-accelerator), with two or more EKS clusters in multiple AWS accounts and verify the collected metrics from all the clusters in the dashboards of a common `Amazon Managed Grafana` workspace in a central monitoring account. + +## Prerequisites + +#### 1. Cross Account IAM access + +In order to create/modify resources across multiple AWS accounts, this Terraform example implements the cross-account IAM role assumption. You will need separate IAM roles in all 3 AWS accounts, and each of these IAM roles should have the below specified trust-relationship so that your local AWS user/role will be able to assume them during the terraform execution. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "" + }, + "Action": "sts:AssumeRole", + "Condition": {} + } + ] +} +``` + +!!! note + The IAM roles in Account 1 and Account 2 (EKS cluster accounts) should have permissions to perform kubernetes API operations against your EKS clusters. For more info, please review documentation for [enabling IAM principal access to your clusters](https://docs.aws.amazon.com/eks/latest/userguide/add-user-role.html) + +#### 2. EKS clusters in multiple AWS Accounts + +Using the example [eks-cluster-with-vpc](https://aws-observability.github.io/terraform-aws-observability-accelerator/helpers/new-eks-cluster/), create two EKS clusters with the below names in two different AWS accounts: + +1. `eks-cluster-1` (Account 1) + +2. `eks-cluster-2` (Account 2) + +Update the cluster names and their corresponding region names in the `variables.tf` file along with the corresponding IAM role ARNs that can be assumed by terraform to perform cross-account API operations. + +#### 3. Amazon Managed Grafana (AMG) workspace + +To run this example you need an existing Amazon Managed Grafana (AMG) workspace. If not, you can create a new AMG workspace by following the [Getting Started with Amazon Managed Grafana](https://docs.aws.amazon.com/grafana/latest/userguide/getting-started-with-AMG.html) documentation. + +Add the Grafana Workspace ID and its corresponding region name in the `variables.tf` file along with the corresponding IAM role ARN that can be assumed by terraform to perform cross-account API operations. + +!!! note + You can obtain the AMG Workspace ID based on its URL. For the URL `https://g-xyz.grafana-workspace.eu-central-1.amazonaws.com`, the workspace ID would be `g-xyz` + + +## Setup + +#### 1. Download sources and initialize Terraform + + +```sh + +git clone https://github.com/aws-observability/terraform-aws-observability-accelerator.git + +cd terraform-aws-observability-accelerator/examples/eks-cross-account-with-central-amp + +terraform init + +``` + +#### 2. Deploy + +By looking at the `variables.tf`, you will notice there are two EKS clusters targeted for deployment by the names/ids: + +1. `eks-cluster-1` + +2. `eks-cluster-2` + +While installing the observability settings for the EKS cluster specified in variable `cluster_one.name`, Terraform also sets up: + +* Creates an `Amazon Managed Prometheus Workspace` + +* Dashboard folder and files in provided `Amazon Managed Grafana Workspace` + + +!!! warning + To override the defaults, create a `terraform.tfvars` and change the default values of the variables. + + + +Run the following command to deploy + +```sh + +terraform apply --auto-approve + +``` + + + +## Verifying Multi Account Observability + + + +One you have successfully run the above setup, you should be able to see dashboards similar to the images shown below in `Amazon Managed Grafana` workspace. + + + +You will notice that you are able to use the `cluster` dropdown to filter the dashboards to metrics collected from a specific EKS cluster. + +![eks-cross-account-1](https://github.com/veekaly/terraform-aws-observability-accelerator/assets/119073483/96a68eb1-4fb7-4a6b-bd4a-15f4f6ac7565) +![eks-cross-account-2](https://github.com/veekaly/terraform-aws-observability-accelerator/assets/119073483/1373b834-1082-4a63-98b9-2b90fb32eada) + + +## Cleanup + +To clean up entirely, run the following command: + + + +```sh + +terraform destroy --auto-approve + +``` diff --git a/examples/eks-cross-account-with-central-amp/README.md b/examples/eks-cross-account-with-central-amp/README.md new file mode 100644 index 0000000..43b6e82 --- /dev/null +++ b/examples/eks-cross-account-with-central-amp/README.md @@ -0,0 +1,126 @@ +# AWS EKS Cross Account Observability + + + +This example shows how to use the [AWS Observability Accelerator](https://github.com/aws-observability/terraform-aws-observability-accelerator), with two or more EKS cluster in multiple AWS accounts and verify the collected metrics from all the clusters in the dashboards of a common `Amazon Managed Grafana` workspace in a central monitoring account. + + + +## Prerequisites + +#### 1. Cross Account IAM access + +In order to create/modify resources across multiple AWS accounts, this Terraform example implements the cross-account IAM role assumption. You will need separate IAM roles in all 3 AWS accounts, and each of these IAM roles should have the below specified trust-relationship so that your local AWS user/role will be able to assume them during the terraform execution. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "" + }, + "Action": "sts:AssumeRole", + "Condition": {} + } + ] +} +``` + +> [!NOTE] +> The IAM roles in Account 1 and Account 2 (EKS cluster accounts) should have permissions to perform kubernetes API operations against your EKS clusters. For more info, please review documentation for [enabling IAM principal access to your clusters](https://docs.aws.amazon.com/eks/latest/userguide/add-user-role.html) + + +#### 2. EKS clusters in multiple AWS Accounts + +Using the example [eks-cluster-with-vpc](../../examples/eks-cluster-with-vpc/), create two EKS clusters with the below names in two different AWS accounts: + +1. `eks-cluster-1` (Account 1) + +2. `eks-cluster-2` (Account 2) + +Update the cluster names and their corresponding region names in the `variables.tf` file along with the corresponding IAM role ARNs that can be assumed by terraform to perform cross-account API operations. + +#### 3. Amazon Managed Grafana (AMG) workspace + +To run this example you need an existing Amazon Managed Grafana (AMG) workspace. If not, you can create a new AMG workspace by following the [Getting Started with Amazon Managed Grafana](https://docs.aws.amazon.com/grafana/latest/userguide/getting-started-with-AMG.html) documentation. + +Add the Grafana Workspace ID and its corresponding region name in the `variables.tf` file along with the corresponding IAM role ARN that can be assumed by terraform to perform cross-account API operations. + +!!! note + +You can obtain the AMG Workspace ID based on its URL. For the URL `https://g-xyz.grafana-workspace.eu-central-1.amazonaws.com`, the workspace ID would be `g-xyz` + + + ## Setup + +#### 1. Download sources and initialize Terraform + + +```sh + +git clone https://github.com/aws-observability/terraform-aws-observability-accelerator.git + +cd terraform-aws-observability-accelerator/examples/eks-cross-account-with-central-amp + +terraform init + +``` + +#### 2. Deploy + +By looking at the `variables.tf`, you will notice there are two EKS clusters targeted for deployment by the names/ids: + +1. `eks-cluster-1` + +2. `eks-cluster-2` + +While installing the observability settings for the EKS cluster specified in variable `cluster_one.name`, Terraform also sets up: + +* Creates an `Amazon Managed Prometheus Workspace` + +* Dashboard folder and files in provided `Amazon Managed Grafana Workspace` + + +!!! warning + +To override the defaults, create a `terraform.tfvars` and change the default values of the variables. + + + +Run the following command to deploy + +```sh + +terraform apply --auto-approve + +``` + + + +## Verifying Multi Account Observability + + + +One you have successfully run the above setup, you should be able to see dashboards similar to the images shown below in `Amazon Managed Grafana` workspace. + + + +You will notice that you are able to use the `cluster` dropdown to filter the dashboards to metrics collected from a specific EKS cluster. + +![eks-cross-account-1](https://github.com/veekaly/terraform-aws-observability-accelerator/assets/119073483/96a68eb1-4fb7-4a6b-bd4a-15f4f6ac7565) +![eks-cross-account-2](https://github.com/veekaly/terraform-aws-observability-accelerator/assets/119073483/1373b834-1082-4a63-98b9-2b90fb32eada) + + +## Cleanup + +To clean up entirely, run the following command: + + + +```sh + +terraform destroy --auto-approve + +``` diff --git a/examples/eks-cross-account-with-central-amp/data.tf b/examples/eks-cross-account-with-central-amp/data.tf new file mode 100644 index 0000000..e87543f --- /dev/null +++ b/examples/eks-cross-account-with-central-amp/data.tf @@ -0,0 +1,19 @@ +data "aws_eks_cluster_auth" "eks_one" { + name = var.cluster_one.name + provider = aws.eks_cluster_one +} + +data "aws_eks_cluster_auth" "eks_two" { + name = var.cluster_two.name + provider = aws.eks_cluster_two +} + +data "aws_eks_cluster" "eks_one" { + name = var.cluster_one.name + provider = aws.eks_cluster_one +} + +data "aws_eks_cluster" "eks_two" { + name = var.cluster_two.name + provider = aws.eks_cluster_two +} diff --git a/examples/eks-cross-account-with-central-amp/iam.tf b/examples/eks-cross-account-with-central-amp/iam.tf new file mode 100644 index 0000000..9a5c193 --- /dev/null +++ b/examples/eks-cross-account-with-central-amp/iam.tf @@ -0,0 +1,73 @@ +data "aws_caller_identity" "monitoring" { + provider = aws.central_monitoring +} + +resource "aws_iam_policy" "irsa_assume_role_policy_one" { + provider = aws.eks_cluster_one + name = "${var.cluster_one.name}-irsa_assume_role_policy" + path = "/" + description = "This role allows the IRSA role to assume the cross-account role for AMP access" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "sts:AssumeRole", + ] + Effect = "Allow" + Resource = "arn:aws:iam::${data.aws_caller_identity.monitoring.account_id}:role/${local.amp_workspace_alias}-role-for-cross-account" + }, + ] + }) +} + +resource "aws_iam_policy" "irsa_assume_role_policy_two" { + provider = aws.eks_cluster_two + name = "${var.cluster_two.name}-irsa_assume_role_policy" + path = "/" + description = "This role allows the IRSA role to assume the cross-account role for AMP access" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "sts:AssumeRole", + ] + Effect = "Allow" + Resource = "arn:aws:iam::${data.aws_caller_identity.monitoring.account_id}:role/${local.amp_workspace_alias}-role-for-cross-account" + }, + ] + }) +} + +resource "aws_iam_role" "cross_account_amp_role" { + provider = aws.central_monitoring + name = "${local.amp_workspace_alias}-role-for-cross-account" + + assume_role_policy = < [grafana\_url](#input\_grafana\_url) | Endpoint URL of Amazon Managed Grafana workspace. Required if `enable_grafana_operator = true` | `string` | `""` | no | | [grafana\_workloads\_dashboard\_url](#input\_grafana\_workloads\_dashboard\_url) | Dashboard URL for Workloads Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/workloads.json"` | no | | [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no | +| [irsa\_iam\_additional\_policies](#input\_irsa\_iam\_additional\_policies) | IAM additional policies for IRSA roles | `list(string)` | `[]` | no | | [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `null` | no | | [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no | | [istio\_config](#input\_istio\_config) | Configuration object for ISTIO monitoring |
object({
enable_alerting_rules = bool
enable_recording_rules = bool
enable_dashboards = bool
scrape_sample_limit = number

flux_gitrepository_name = string
flux_gitrepository_url = string
flux_gitrepository_branch = string
flux_kustomization_name = string
flux_kustomization_path = string

managed_prometheus_workspace_id = string
prometheus_metrics_endpoint = string

dashboards = object({
cp = string
mesh = string
performance = string
service = string
})
})
| `null` | no | @@ -116,6 +117,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [ksm\_config](#input\_ksm\_config) | Kube State metrics configuration |
object({
create_namespace = bool
k8s_namespace = string
helm_chart_name = string
helm_chart_version = string
helm_release_name = string
helm_repo_url = string
helm_settings = map(string)
helm_values = map(any)

scrape_interval = string
scrape_timeout = string
})
|
{
"create_namespace": true,
"helm_chart_name": "kube-state-metrics",
"helm_chart_version": "4.24.0",
"helm_release_name": "kube-state-metrics",
"helm_repo_url": "https://prometheus-community.github.io/helm-charts",
"helm_settings": {},
"helm_values": {},
"k8s_namespace": "kube-system",
"scrape_interval": "60s",
"scrape_timeout": "15s"
}
| no | | [kubeproxy\_monitoring\_config](#input\_kubeproxy\_monitoring\_config) | Config object for kube-proxy monitoring |
object({
flux_gitrepository_name = string
flux_gitrepository_url = string
flux_gitrepository_branch = string
flux_kustomization_name = string
flux_kustomization_path = string

dashboards = object({
grafana_kubeproxy_dashboard_url = string
})
})
| `null` | no | | [logs\_config](#input\_logs\_config) | Configuration object for logs collection |
object({
cw_log_retention_days = number
})
|
{
"cw_log_retention_days": 90
}
| no | +| [managed\_prometheus\_cross\_account\_role](#input\_managed\_prometheus\_cross\_account\_role) | Amazon Managed Prometheus Workspace's Account Role Arn | `string` | `""` | no | | [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | | [managed\_prometheus\_workspace\_region](#input\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus Workspace's Region | `string` | `null` | no | @@ -131,6 +133,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | Name | Description | |------|-------------| +| [adot\_irsa\_arn](#output\_adot\_irsa\_arn) | IRSA Arn for ADOT | | [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id | | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 4b7e9d4..b80e86b 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -93,6 +93,10 @@ module "helm_addon" { name = "region" value = var.managed_prometheus_workspace_region }, + { + name = "assumeRoleArn" + value = var.managed_prometheus_cross_account_role + }, { name = "ekscluster" value = local.context.eks_cluster_id @@ -193,10 +197,11 @@ module "helm_addon" { kubernetes_namespace = local.namespace create_kubernetes_service_account = true kubernetes_service_account = try(var.helm_config.service_account, local.name) - irsa_iam_policies = [ + irsa_iam_policies = flatten([ "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess", - "arn:${data.aws_partition.current.partition}:iam::aws:policy/AWSXrayWriteOnlyAccess" - ] + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AWSXrayWriteOnlyAccess", + var.irsa_iam_additional_policies, + ]) } addon_context = local.context diff --git a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml index 10f561c..22b9c3a 100644 --- a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml +++ b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml @@ -1497,6 +1497,11 @@ spec: sigv4auth: region: {{ .Values.region }} service: aps + {{ if ne .Values.assumeRoleArn "" }} + assume_role: + arn: {{ .Values.assumeRoleArn }} + sts_region: {{ .Values.region }} + {{ end }} health_check: pprof: endpoint: :1888 diff --git a/modules/eks-monitoring/otel-config/values.yaml b/modules/eks-monitoring/otel-config/values.yaml index 5d74367..263ce21 100644 --- a/modules/eks-monitoring/otel-config/values.yaml +++ b/modules/eks-monitoring/otel-config/values.yaml @@ -3,6 +3,8 @@ region: ${region} ekscluster: ${eks_cluster} accountId: ${account_id} +assumeRoleArn: ${managed_prometheus_cross_account_role} + globalScrapeTimeout: ${global_scrape_timeout} globalScrapeSampleLimit: ${global_scrape_sample_limit} diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index a68742f..c85d3cc 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -7,3 +7,8 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = var.eks_cluster_id } + +output "adot_irsa_arn" { + description = "IRSA Arn for ADOT" + value = module.helm_addon.irsa_arn +} diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index 1b047f1..c6d7483 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -33,6 +33,12 @@ variable "irsa_iam_permissions_boundary" { default = null } +variable "irsa_iam_additional_policies" { + description = "IAM additional policies for IRSA roles" + type = list(string) + default = [] +} + variable "adot_loglevel" { description = "Verbosity level for ADOT collector logs. This accepts (detailed|normal|basic), see https://aws-otel.github.io/docs/components/misc-exporters for mor infos." type = string @@ -57,6 +63,12 @@ variable "managed_prometheus_workspace_region" { default = null } +variable "managed_prometheus_cross_account_role" { + description = "Amazon Managed Prometheus Workspace's Account Role Arn" + type = string + default = "" +} + variable "enable_alerting_rules" { description = "Enables or disables Managed Prometheus alerting rules" type = bool