From c726f508b59ea54365189cb05d94084dc02d2b9d Mon Sep 17 00:00:00 2001 From: Rodrigue Koffi Date: Thu, 19 Jan 2023 21:24:39 +0100 Subject: [PATCH] Tracing support with custom metrics collection (#67) * Init tracing support * Update xray exporter * Remove duplicate metrics only on daemon set * Add support for custom metrics collection * Remove experimental feature * Rename config items * Update requirements for tracing support --- .github/workflows/docbuild.yml | 7 +- modules/workloads/infra/README.md | 3 + modules/workloads/infra/main.tf | 29 ++- .../templates/opentelemetrycollector.yaml | 171 +++++++++++++++++- .../workloads/infra/otel-config/values.yaml | 41 +---- modules/workloads/infra/variables.tf | 27 +++ 6 files changed, 236 insertions(+), 42 deletions(-) diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml index a99afb4..17c7f99 100644 --- a/.github/workflows/docbuild.yml +++ b/.github/workflows/docbuild.yml @@ -1,8 +1,8 @@ -name: ci +name: ci on: push: branches: - - master + - master - main permissions: contents: write @@ -14,6 +14,5 @@ jobs: - uses: actions/setup-python@v4 with: python-version: 3.x - - run: pip install mkdocs-material + - run: pip install mkdocs-material - run: mkdocs gh-deploy --force - diff --git a/modules/workloads/infra/README.md b/modules/workloads/infra/README.md index fabd4d6..18c27cf 100644 --- a/modules/workloads/infra/README.md +++ b/modules/workloads/infra/README.md @@ -57,13 +57,16 @@ This module is inspired from the open source [kube-prometheus-stack](https://git | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [custom\_metrics\_config](#input\_custom\_metrics\_config) | Configuration object to enable custom metrics collection | 
object({
    ports = list(number)
    # paths = optional(list(string), ["/metrics"])
    # list of samples to be dropped by label prefix, ex: go_ -> discards go_.*
    dropped_series_prefixes = list(string)
  })
| 
{
  "dropped_series_prefixes": [
    "unspecified"
  ],
  "ports": []
}
| no | | [dashboards\_folder\_id](#input\_dashboards\_folder\_id) | Grafana folder ID for automatic dashboards | `string` | n/a | yes | | [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes | | [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no | +| [enable\_custom\_metrics](#input\_enable\_custom\_metrics) | Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included | `bool` | `false` | no | | [enable\_dashboards](#input\_enable\_dashboards) | Enables or disables curated dashboards | `bool` | `true` | no | | [enable\_kube\_state\_metrics](#input\_enable\_kube\_state\_metrics) | Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no | | [enable\_node\_exporter](#input\_enable\_node\_exporter) | Enables or disables Node exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no | | [enable\_recording\_rules](#input\_enable\_recording\_rules) | Enables or disables Managed Prometheus recording rules. Disabling this might affect some data in the dashboards | `bool` | `true` | no | +| [enable\_tracing](#input\_enable\_tracing) | (Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0 | `bool` | `false` | no | | [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no | | [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `""` | no | | [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no | diff --git a/modules/workloads/infra/main.tf b/modules/workloads/infra/main.tf index 7b5ec1f..6b6dd30 100644 --- a/modules/workloads/infra/main.tf +++ b/modules/workloads/infra/main.tf @@ -73,6 +73,30 @@ module "helm_addon" { name = "accountId" value = local.context.aws_caller_identity_account_id }, + { + name = "enableTracing" + value = var.enable_tracing + }, + { + name = "otlpHttpEndpoint" + value = "0.0.0.0:4318" + }, + { + name = "otlpGrpcEndpoint" + value = "0.0.0.0:4317" + }, + { + name = "enableCustomMetrics" + value = var.enable_custom_metrics + }, + { + name = "customMetricsPorts" + value = format(".*:(%s)$", join("|", var.custom_metrics_config.ports)) + }, + { + name = "customMetricsDroppedSeriesPrefixes" + value = format("(%s.*)$", join(".*|", var.custom_metrics_config.dropped_series_prefixes)) + } ] irsa_config = { @@ -80,7 +104,10 @@ module "helm_addon" { kubernetes_namespace = local.namespace create_kubernetes_service_account = true kubernetes_service_account = try(var.helm_config.service_account, local.name) - irsa_iam_policies = ["arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess"] + irsa_iam_policies = [ + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AWSXrayWriteOnlyAccess" + ] } addon_context = local.context diff --git a/modules/workloads/infra/otel-config/templates/opentelemetrycollector.yaml b/modules/workloads/infra/otel-config/templates/opentelemetrycollector.yaml index 10088ad..ed42cfa 100644 --- a/modules/workloads/infra/otel-config/templates/opentelemetrycollector.yaml +++ b/modules/workloads/infra/otel-config/templates/opentelemetrycollector.yaml @@ -3,11 +3,42 @@ kind: OpenTelemetryCollector metadata: name: adot spec: - image: public.ecr.aws/aws-observability/aws-otel-collector:v0.21.0 + image: public.ecr.aws/aws-observability/aws-otel-collector:v0.22.1 + {{ if .Values.enableTracing }} + mode: daemonset + hostNetwork: true + ports: + - port: 4317 + name: "otlpgrpc" + - port: 4318 + name: "oltphttp" + {{ else }} mode: deployment + {{ end }} serviceAccount: adot-collector-kubeprometheus + env: + - name: "K8S_NODE_NAME" + valueFrom: + fieldRef: + fieldPath: "spec.nodeName" + - name: "K8S_POD_NAME" + valueFrom: + fieldRef: + fieldPath: "metadata.name" + - name: "K8S_NAMESPACE" + valueFrom: + fieldRef: + fieldPath: "metadata.namespace" config: | receivers: + {{ if .Values.enableTracing }} + otlp: + protocols: + grpc: + endpoint: {{ .Values.otlpGrpcEndpoint }} + http: + endpoint: {{ .Values.otlpHttpEndpoint }} + {{ end }} prometheus: config: global: @@ -37,6 +68,11 @@ spec: regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$${1}/proxy/metrics + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_node_name] + {{ end }} - job_name: 'kubelet' scheme: https tls_config: @@ -54,6 +90,11 @@ spec: regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_node_name] + {{ end }} - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0 honor_timestamps: true scrape_interval: {{ .Values.globalScrapeInterval }} @@ -148,6 +189,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -250,6 +296,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -351,6 +402,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -468,6 +524,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -585,6 +646,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -701,6 +767,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -805,6 +876,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -911,6 +987,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1017,6 +1098,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1123,6 +1209,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1229,6 +1320,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1335,6 +1431,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1437,6 +1538,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1539,6 +1645,11 @@ spec: regex: "0" replacement: $$1 action: keep + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} kubernetes_sd_configs: - role: endpoints kubeconfig_file: "" @@ -1562,13 +1673,53 @@ spec: - action: replace source_labels: [__meta_kubernetes_endpoint_node_name] target_label: nodename + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_endpoint_node_name] + {{ end }} + - job_name: "custom-metrics" + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [ __address__ ] + action: keep + regex: '{{ .Values.customMetricsPorts }}' + - action: replace + source_labels: [__meta_kubernetes_pod_node_name] + target_label: nodename + - action: replace + source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - action: replace + source_labels: [__meta_kubernetes_pod_name] + target_label: pod_name + - action: replace + source_labels: [__meta_kubernetes_pod_container_name] + target_label: container_name + - action: replace + source_labels: [__meta_kubernetes_pod_controller_kind] + target_label: pod_controller_kind + {{ if .Values.enableTracing }} + - action: keep + regex: $K8S_NODE_NAME + source_labels: [__meta_kubernetes_pod_node_name] + {{ end }} + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: '{{ .Values.customMetricsDroppedSeriesPrefixes }}' + action: drop exporters: + {{ if .Values.enableTracing }} + awsxray: + region: {{ .Values.region }} + {{ end }} prometheusremotewrite: endpoint: {{ .Values.ampurl }} auth: authenticator: sigv4auth logging: - loglevel: info + loglevel: warn extensions: sigv4auth: region: {{ .Values.region }} @@ -1578,9 +1729,25 @@ spec: endpoint: :1888 zpages: endpoint: :55679 + processors: + batch/metrics: + timeout: 30s + send_batch_size: 500 + {{ if .Values.enableTracing }} + batch/traces: + timeout: 10s + send_batch_size: 50 + {{ end }} service: extensions: [pprof, zpages, health_check, sigv4auth] pipelines: metrics: receivers: [prometheus] + processors: [batch/metrics] exporters: [logging, prometheusremotewrite] + {{ if .Values.enableTracing }} + traces: + receivers: [otlp] + processors: [batch/traces] + exporters: [logging, awsxray] + {{ end }} diff --git a/modules/workloads/infra/otel-config/values.yaml b/modules/workloads/infra/otel-config/values.yaml index 10c83c5..db82a5c 100644 --- a/modules/workloads/infra/otel-config/values.yaml +++ b/modules/workloads/infra/otel-config/values.yaml @@ -4,40 +4,11 @@ ekscluster: ${eks_cluster} globalScrapeTimeout: ${global_scrape_timeout} globalScrapeSampleLimit: ${global_scrape_sample_limit} -# TODO: enable after terraform 1.3 as defaults will be optional -#nodeExporterScrapeInterval: ${node_exporter_scrape_interval} -#nodeExporterScrapeTimeout: ${node_exporter_scrape_timeout} +enableTracing: ${enable_tracing} +otlpGrpcEndpoint: ${otlp_grpc_endpoint} +otlpHttpEndpoint: ${otlp_http_endpoint} -#kubeletScrapeInterval: ${kubelet_scrape_interval} -#kubeletScrapeTimeout: ${kubelet_scrape_timeout} - -#operatorScrapeInterval: ${operator_scrape_interval} -#operatorScrapeTimeout: ${operator_scrape_timeout} - -#kubeletScrapeInterval: ${operator_scrape_interval} -#kubeletScrapeTimeout: ${operator_scrape_timeout} - -#ksmScrapeInterval: ${ksm_scrape_interval} -#ksmScrapeTimeout: ${ksm_scrape_timeout} - -#schedulerScrapeInterval: ${scheduler_scrape_interval} -#schedulerScrapeTimeout: ${scheduler_scrape_timeout} - -#proxyScrapeInterval: ${proxy_scrape_interval} -#proxyScrapeTimeout: ${proxy_scrape_timeout} - -#etcdScrapeInterval: ${etcd_scrape_interval} -#etcdScrapeTimeout: ${etcd_scrape_timeout} - -#controllerManagerScrapeInterval: ${controller_manager_scrape_interval} -#controllerManagerScrapeTimeout: ${controller_manager_scrape_timeout} - -#corednsScrapeInterval: ${coredns_scrape_interval} -#corednsScrapeTimeout: ${coredns_scrape_timeout} - -#apiserverScrapeInterval: ${apiserver_scrape_interval} -#apiserverScrapeTimeout: ${apiserver_scrape_timeout} - -#alertmanagerScrapeInterval: ${alertmnager_scrape_interval} -#alertmanagerScrapeTimeout: ${alertmnager_scrape_timeout} +enableCustomMetrics: ${enable_custom_metrics} +customMetricsPorts: ${custom_metrics_ports} +customMetricsDroppedSeriesPrefixes: ${custom_metrics_dropped_series_prefixes} diff --git a/modules/workloads/infra/variables.tf b/modules/workloads/infra/variables.tf index 7bed974..c077832 100644 --- a/modules/workloads/infra/variables.tf +++ b/modules/workloads/infra/variables.tf @@ -156,3 +156,30 @@ variable "prometheus_config" { } nullable = false } + +variable "enable_tracing" { + description = "(Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0" + type = bool + default = false +} + +variable "enable_custom_metrics" { + description = "Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included" + type = bool + default = false +} + +variable "custom_metrics_config" { + description = "Configuration object to enable custom metrics collection" + type = object({ + ports = list(number) + # paths = optional(list(string), ["/metrics"]) + # list of samples to be dropped by label prefix, ex: go_ -> discards go_.* + dropped_series_prefixes = list(string) + }) + + default = { + ports = [] + dropped_series_prefixes = ["unspecified"] + } +}