storytold/terraform-aws-observability-accelerator
2
0
Fork 0
mirror of https://github.com/storytold/terraform-aws-observability-accelerator.git synced 2026-10-27 08:36:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tracing support with custom metrics collection (#67)

Browse Source 
* Init tracing support

* Update xray exporter

* Remove duplicate metrics only on daemon set

* Add support for custom metrics collection

* Remove experimental feature

* Rename config items

* Update requirements for tracing support
This commit is contained in:
Rodrigue Koffi
2023-01-19 21:24:39 +01:00
committed by GitHub
parent e73b4b9300
commit c726f508b5
6 changed files with 236 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/workloads/infra/otel-config/templates/opentelemetrycollector.yaml
+169 -2
View File
@@ -3,11 +3,42 @@ kind: OpenTelemetryCollector
metadata:
name: adot
spec:
image: public.ecr.aws/aws-observability/aws-otel-collector:v0.21.0
image: public.ecr.aws/aws-observability/aws-otel-collector:v0.22.1
{{ if .Values.enableTracing }}
mode: daemonset
hostNetwork: true
ports:
- port: 4317
name: "otlpgrpc"
- port: 4318
name: "oltphttp"
{{ else }}
mode: deployment
{{ end }}
serviceAccount: adot-collector-kubeprometheus
env:
- name: "K8S_NODE_NAME"
valueFrom:
fieldRef:
fieldPath: "spec.nodeName"
- name: "K8S_POD_NAME"
valueFrom:
fieldRef:
fieldPath: "metadata.name"
- name: "K8S_NAMESPACE"
valueFrom:
fieldRef:
fieldPath: "metadata.namespace"
config: |
receivers:
{{ if .Values.enableTracing }}
otlp:
protocols:
grpc:
endpoint: {{ .Values.otlpGrpcEndpoint }}
http:
endpoint: {{ .Values.otlpHttpEndpoint }}
{{ end }}
prometheus:
config:
global:
@@ -37,6 +68,11 @@ spec:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$${1}/proxy/metrics
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_node_name]
{{ end }}
- job_name: 'kubelet'
scheme: https
tls_config:
@@ -54,6 +90,11 @@ spec:
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_node_name]
{{ end }}
- job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0
honor_timestamps: true
scrape_interval: {{ .Values.globalScrapeInterval }}
@@ -148,6 +189,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -250,6 +296,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -351,6 +402,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -468,6 +524,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -585,6 +646,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -701,6 +767,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -805,6 +876,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -911,6 +987,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1017,6 +1098,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1123,6 +1209,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1229,6 +1320,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1335,6 +1431,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1437,6 +1538,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1539,6 +1645,11 @@ spec:
regex: "0"
replacement: $$1
action: keep
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: ""
@@ -1562,13 +1673,53 @@ spec:
- action: replace
source_labels: [__meta_kubernetes_endpoint_node_name]
target_label: nodename
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_endpoint_node_name]
{{ end }}
- job_name: "custom-metrics"
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [ __address__ ]
action: keep
regex: '{{ .Values.customMetricsPorts }}'
- action: replace
source_labels: [__meta_kubernetes_pod_node_name]
target_label: nodename
- action: replace
source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- action: replace
source_labels: [__meta_kubernetes_pod_name]
target_label: pod_name
- action: replace
source_labels: [__meta_kubernetes_pod_container_name]
target_label: container_name
- action: replace
source_labels: [__meta_kubernetes_pod_controller_kind]
target_label: pod_controller_kind
{{ if .Values.enableTracing }}
- action: keep
regex: $K8S_NODE_NAME
source_labels: [__meta_kubernetes_pod_node_name]
{{ end }}
metric_relabel_configs:
- source_labels: [ __name__ ]
regex: '{{ .Values.customMetricsDroppedSeriesPrefixes }}'
action: drop
exporters:
{{ if .Values.enableTracing }}
awsxray:
region: {{ .Values.region }}
{{ end }}
prometheusremotewrite:
endpoint: {{ .Values.ampurl }}
auth:
authenticator: sigv4auth
logging:
loglevel: info
loglevel: warn
extensions:
sigv4auth:
region: {{ .Values.region }}
@@ -1578,9 +1729,25 @@ spec:
endpoint: :1888
zpages:
endpoint: :55679
processors:
batch/metrics:
timeout: 30s
send_batch_size: 500
{{ if .Values.enableTracing }}
batch/traces:
timeout: 10s
send_batch_size: 50
{{ end }}
service:
extensions: [pprof, zpages, health_check, sigv4auth]
pipelines:
metrics:
receivers: [prometheus]
processors: [batch/metrics]
exporters: [logging, prometheusremotewrite]
{{ if .Values.enableTracing }}
traces:
receivers: [otlp]
processors: [batch/traces]
exporters: [logging, awsxray]
{{ end }}
modules/workloads/infra/otel-config/values.yaml
+6 -35
View File
@@ -4,40 +4,11 @@ ekscluster: ${eks_cluster}
globalScrapeTimeout: ${global_scrape_timeout}
globalScrapeSampleLimit: ${global_scrape_sample_limit}
# TODO: enable after terraform 1.3 as defaults will be optional
#nodeExporterScrapeInterval: ${node_exporter_scrape_interval}
#nodeExporterScrapeTimeout: ${node_exporter_scrape_timeout}
enableTracing: ${enable_tracing}
otlpGrpcEndpoint: ${otlp_grpc_endpoint}
otlpHttpEndpoint: ${otlp_http_endpoint}
#kubeletScrapeInterval: ${kubelet_scrape_interval}
#kubeletScrapeTimeout: ${kubelet_scrape_timeout}
#operatorScrapeInterval: ${operator_scrape_interval}
#operatorScrapeTimeout: ${operator_scrape_timeout}
#kubeletScrapeInterval: ${operator_scrape_interval}
#kubeletScrapeTimeout: ${operator_scrape_timeout}
#ksmScrapeInterval: ${ksm_scrape_interval}
#ksmScrapeTimeout: ${ksm_scrape_timeout}
#schedulerScrapeInterval: ${scheduler_scrape_interval}
#schedulerScrapeTimeout: ${scheduler_scrape_timeout}
#proxyScrapeInterval: ${proxy_scrape_interval}
#proxyScrapeTimeout: ${proxy_scrape_timeout}
#etcdScrapeInterval: ${etcd_scrape_interval}
#etcdScrapeTimeout: ${etcd_scrape_timeout}
#controllerManagerScrapeInterval: ${controller_manager_scrape_interval}
#controllerManagerScrapeTimeout: ${controller_manager_scrape_timeout}
#corednsScrapeInterval: ${coredns_scrape_interval}
#corednsScrapeTimeout: ${coredns_scrape_timeout}
#apiserverScrapeInterval: ${apiserver_scrape_interval}
#apiserverScrapeTimeout: ${apiserver_scrape_timeout}
#alertmanagerScrapeInterval: ${alertmnager_scrape_interval}
#alertmanagerScrapeTimeout: ${alertmnager_scrape_timeout}
enableCustomMetrics: ${enable_custom_metrics}
customMetricsPorts: ${custom_metrics_ports}
customMetricsDroppedSeriesPrefixes: ${custom_metrics_dropped_series_prefixes}