storytold/terraform-aws-observability-accelerator
2
0
Fork 0
mirror of https://github.com/storytold/terraform-aws-observability-accelerator.git synced 2026-10-09 00:09:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tracing support with custom metrics collection (#67)

Browse Source 
* Init tracing support

* Update xray exporter

* Remove duplicate metrics only on daemon set

* Add support for custom metrics collection

* Remove experimental feature

* Rename config items

* Update requirements for tracing support
This commit is contained in:
Rodrigue Koffi
2023-01-19 21:24:39 +01:00
committed by GitHub
parent e73b4b9300
commit c726f508b5
6 changed files with 236 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/workloads/infra/README.md
+3
View File
@@ -57,13 +57,16 @@ This module is inspired from the open source [kube-prometheus-stack](https://git
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_custom_metrics_config"></a> [custom\_metrics\_config](#input\_custom\_metrics\_config) | Configuration object to enable custom metrics collection | <pre>object({<br> ports = list(number)<br> # paths = optional(list(string), ["/metrics"])<br> # list of samples to be dropped by label prefix, ex: go_ -> discards go_.*<br> dropped_series_prefixes = list(string)<br> })</pre> | <pre>{<br> "dropped_series_prefixes": [<br> "unspecified"<br> ],<br> "ports": []<br>}</pre> | no |
| <a name="input_dashboards_folder_id"></a> [dashboards\_folder\_id](#input\_dashboards\_folder\_id) | Grafana folder ID for automatic dashboards | `string` | n/a | yes |
| <a name="input_eks_cluster_id"></a> [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes |
| <a name="input_enable_alerting_rules"></a> [enable\_alerting\_rules](#input\_enable\_alerting\_rules) | Enables or disables Managed Prometheus alerting rules | `bool` | `true` | no |
| <a name="input_enable_custom_metrics"></a> [enable\_custom\_metrics](#input\_enable\_custom\_metrics) | Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included | `bool` | `false` | no |
| <a name="input_enable_dashboards"></a> [enable\_dashboards](#input\_enable\_dashboards) | Enables or disables curated dashboards | `bool` | `true` | no |
| <a name="input_enable_kube_state_metrics"></a> [enable\_kube\_state\_metrics](#input\_enable\_kube\_state\_metrics) | Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
| <a name="input_enable_node_exporter"></a> [enable\_node\_exporter](#input\_enable\_node\_exporter) | Enables or disables Node exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
| <a name="input_enable_recording_rules"></a> [enable\_recording\_rules](#input\_enable\_recording\_rules) | Enables or disables Managed Prometheus recording rules. Disabling this might affect some data in the dashboards | `bool` | `true` | no |
| <a name="input_enable_tracing"></a> [enable\_tracing](#input\_enable\_tracing) | (Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0 | `bool` | `false` | no |
| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm Config for Prometheus | `any` | `{}` | no |
| <a name="input_irsa_iam_permissions_boundary"></a> [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `""` | no |
| <a name="input_irsa_iam_role_path"></a> [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no |
modules/workloads/infra/main.tf
+28 -1
View File
@@ -73,6 +73,30 @@ module "helm_addon" {
name = "accountId"
value = local.context.aws_caller_identity_account_id
},
{
name = "enableTracing"
value = var.enable_tracing
},
{
name = "otlpHttpEndpoint"
value = "0.0.0.0:4318"
},
{
name = "otlpGrpcEndpoint"
value = "0.0.0.0:4317"
},
{
name = "enableCustomMetrics"
value = var.enable_custom_metrics
},
{
name = "customMetricsPorts"
value = format(".*:(%s)$", join("|", var.custom_metrics_config.ports))
},
{
name = "customMetricsDroppedSeriesPrefixes"
value = format("(%s.*)$", join(".*|", var.custom_metrics_config.dropped_series_prefixes))
}
]
irsa_config = {
@@ -80,7 +104,10 @@ module "helm_addon" {
kubernetes_namespace = local.namespace
create_kubernetes_service_account = true
kubernetes_service_account = try(var.helm_config.service_account, local.name)
irsa_iam_policies = ["arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess"]
irsa_iam_policies = [
"arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonPrometheusRemoteWriteAccess",
"arn:${data.aws_partition.current.partition}:iam::aws:policy/AWSXrayWriteOnlyAccess"
]
}
addon_context = local.context
modules/workloads/infra/otel-config/templates/opentelemetrycollector.yaml
+169 -2
View File
@@ -3,11 +3,42 @@ kind: OpenTelemetryCollector
metadata:
name: adot
spec:
image: public.ecr.aws/aws-observability/aws-otel-collector:v0.21.0
image: public.ecr.aws/aws-observability/aws-otel-collector:v0.22.1
{{ if .Values.enableTracing }}
mode: daemonset
hostNetwork: true
ports:
- port: 4317
name: "otlpgrpc"
- port: 4318
name: "oltphttp"
{{ else }}
mode: deployment
{{ end }}
serviceAccount: adot-collector-kubeprometheus
env:
- name: "K8S_NODE_NAME"
valueFrom:
fieldRef:
fieldPath: "spec.nodeName"
- name: "K8S_POD_NAME"
valueFrom:
fieldRef:
fieldPath: "metadata.name"
- name: "K8S_NAMESPACE"
valueFrom:
fieldRef:
fieldPath: "metadata.namespace"
config: |
receivers:
{{ if .Values.enableTracing }}
otlp:
protocols:
grpc:
endpoint: {{ .Values.otlpGrpcEndpoint }}
http:
endpoint: {{ .Values.otlpHttpEndpoint }}
{{ end }}
prometheus:
config:
global:
@@ -37,6 +68,11 @@ spec:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$${1}/proxy/metrics
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_node_name]
{{ end }}
- job_name: 'kubelet'
scheme: https
tls_config:
@@ -54,6 +90,11 @@ spec:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_node_name]
{{ end }}
- job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0
honor_timestamps: true
scrape_interval: {{ .Values.globalScrapeInterval }}
@@ -148,6 +189,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -250,6 +296,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -351,6 +402,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -468,6 +524,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -585,6 +646,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -701,6 +767,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -805,6 +876,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -911,6 +987,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1017,6 +1098,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1123,6 +1209,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1229,6 +1320,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1335,6 +1431,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1437,6 +1538,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1539,6 +1645,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1562,13 +1673,53 @@ spec:
- action: replace
source_labels: [__meta_kubernetes_endpoint_node_name]
target_label: nodename
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
- job_name: "custom-metrics"
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [ __address__ ]
action: keep
regex: '{{ .Values.customMetricsPorts }}'
- action: replace
source_labels: [__meta_kubernetes_pod_node_name]
target_label: nodename
- action: replace
source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- action: replace
source_labels: [__meta_kubernetes_pod_name]
target_label: pod_name
- action: replace
source_labels: [__meta_kubernetes_pod_container_name]
target_label: container_name
- action: replace
source_labels: [__meta_kubernetes_pod_controller_kind]
target_label: pod_controller_kind
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_pod_node_name]
{{ end }}
metric_relabel_configs:
- source_labels: [ __name__ ]
regex: '{{ .Values.customMetricsDroppedSeriesPrefixes }}'
action: drop
exporters:
{{ if .Values.enableTracing }}
awsxray:
region: {{ .Values.region }}
{{ end }}
prometheusremotewrite:
endpoint: {{ .Values.ampurl }}
auth:
authenticator: sigv4auth
logging:
loglevel: info
loglevel: warn
extensions:
sigv4auth:
region: {{ .Values.region }}
@@ -1578,9 +1729,25 @@ spec:
endpoint: :1888
zpages:
endpoint: :55679
processors:
batch/metrics:
timeout: 30s
send_batch_size: 500
{{ if .Values.enableTracing }}
batch/traces:
timeout: 10s
send_batch_size: 50
{{ end }}
service:
extensions: [pprof, zpages, health_check, sigv4auth]
pipelines:
metrics:
receivers: [prometheus]
processors: [batch/metrics]
exporters: [logging, prometheusremotewrite]
{{ if .Values.enableTracing }}
traces:
receivers: [otlp]
processors: [batch/traces]
exporters: [logging, awsxray]
{{ end }}
modules/workloads/infra/otel-config/values.yaml
+6 -35
View File
@@ -4,40 +4,11 @@ ekscluster: ${eks_cluster}
globalScrapeTimeout: ${global_scrape_timeout}
globalScrapeSampleLimit: ${global_scrape_sample_limit}
# TODO: enable after terraform 1.3 as defaults will be optional
#nodeExporterScrapeInterval: ${node_exporter_scrape_interval}
#nodeExporterScrapeTimeout: ${node_exporter_scrape_timeout}
enableTracing: ${enable_tracing}
otlpGrpcEndpoint: ${otlp_grpc_endpoint}
otlpHttpEndpoint: ${otlp_http_endpoint}
#kubeletScrapeInterval: ${kubelet_scrape_interval}
#kubeletScrapeTimeout: ${kubelet_scrape_timeout}
#operatorScrapeInterval: ${operator_scrape_interval}
#operatorScrapeTimeout: ${operator_scrape_timeout}
#kubeletScrapeInterval: ${operator_scrape_interval}
#kubeletScrapeTimeout: ${operator_scrape_timeout}
#ksmScrapeInterval: ${ksm_scrape_interval}
#ksmScrapeTimeout: ${ksm_scrape_timeout}
#schedulerScrapeInterval: ${scheduler_scrape_interval}
#schedulerScrapeTimeout: ${scheduler_scrape_timeout}
#proxyScrapeInterval: ${proxy_scrape_interval}
#proxyScrapeTimeout: ${proxy_scrape_timeout}
#etcdScrapeInterval: ${etcd_scrape_interval}
#etcdScrapeTimeout: ${etcd_scrape_timeout}
#controllerManagerScrapeInterval: ${controller_manager_scrape_interval}
#controllerManagerScrapeTimeout: ${controller_manager_scrape_timeout}
#corednsScrapeInterval: ${coredns_scrape_interval}
#corednsScrapeTimeout: ${coredns_scrape_timeout}
#apiserverScrapeInterval: ${apiserver_scrape_interval}
#apiserverScrapeTimeout: ${apiserver_scrape_timeout}
#alertmanagerScrapeInterval: ${alertmnager_scrape_interval}
#alertmanagerScrapeTimeout: ${alertmnager_scrape_timeout}
enableCustomMetrics: ${enable_custom_metrics}
customMetricsPorts: ${custom_metrics_ports}
customMetricsDroppedSeriesPrefixes: ${custom_metrics_dropped_series_prefixes}
modules/workloads/infra/variables.tf
+27
View File
@@ -156,3 +156,30 @@ variable "prometheus_config" {
}
nullable = false
}
variable "enable_tracing" {
description = "(Experimental) Enables tracing with AWS X-Ray. This changes the deploy mode of the collector to daemon set. Requirement: adot add-on <= 0.58-build.0"
type = bool
default = false
}
variable "enable_custom_metrics" {
description = "Allows additional metrics collection for config elements in the `custom_metrics_config` config object. Automatic dashboards are not included"
type = bool
default = false
}
variable "custom_metrics_config" {
description = "Configuration object to enable custom metrics collection"
type = object({
ports = list(number)
# paths = optional(list(string), ["/metrics"])
# list of samples to be dropped by label prefix, ex: go_ -> discards go_.*
dropped_series_prefixes = list(string)
})
default = {
ports = []
dropped_series_prefixes = ["unspecified"]
}
}