// gce node exporter discovery.gce "node_exporter_a" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-a" port = 9100 } discovery.gce "node_exporter_b" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-b" port = 9100 } discovery.gce "node_exporter_c" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-c" port = 9100 } discovery.relabel "node_exporter_gce" { targets = concat( discovery.gce.node_exporter_a.targets, discovery.gce.node_exporter_b.targets, discovery.gce.node_exporter_c.targets, ) rule { source_labels = ["__meta_gce_instance_status"] regex = "RUNNING" action = "keep" } rule { source_labels = ["__meta_gce_label_goog-k8s-cluster-name"] regex = "k8s" action = "drop" } rule { source_labels = ["__meta_gce_label_monitoring"] regex = "no" action = "drop" } rule { source_labels = ["__meta_gce_instance_name"] regex = "gke-k8s-.*" action = "drop" } rule { target_label = "project" replacement = "meesho-datascience-prd-0622" } rule { source_labels = ["__meta_gce_instance_name"] target_label = "name" } rule { source_labels = ["__meta_gce_instance_id"] target_label = "instance_id" } rule { source_labels = ["__meta_gce_instance_status"] target_label = "status" } rule { source_labels = ["__meta_gce_private_ip"] target_label = "private_ip" } rule { source_labels = ["__meta_gce_public_ip"] target_label = "public_ip" } rule { source_labels = ["__meta_gce_label_bu"] target_label = "bu" } rule { source_labels = ["__meta_gce_label_component"] target_label = "component" } rule { source_labels = ["__meta_gce_label_env"] target_label = "env" } rule { source_labels = ["__meta_gce_label_service"] target_label = "service" } rule { source_labels = ["__meta_gce_label_team"] target_label = "team" } rule { source_labels = ["__meta_gce_label_type"] target_label = "type" } rule { source_labels = ["__meta_gce_label_alerts"] target_label = "alerts" } } prometheus.scrape "node_exporter_gce" { targets = discovery.relabel.node_exporter_gce.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "node_exporter_gce" clustering { enabled = true } } // kube state metrics discovery.relabel "kube_state_metrics_pod" { targets = [{__address__ = "kube-state-metrics-datascience-prd.kube-system.svc.cluster.local:8080",}] rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { target_label = "cluster" replacement = "k8s-datascience-prd-ase1" } } prometheus.scrape "kube_state_metrics_pod" { targets = discovery.relabel.kube_state_metrics_pod.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "kube_state_metrics_pod" scrape_interval = "15s" clustering { enabled = true } } // cadvisor metrics discovery.kubernetes "kubernetes_nodes" { role = "node" } discovery.relabel "cadvisor_kubernetes_nodes" { targets = discovery.kubernetes.kubernetes_nodes.targets rule { replacement = "k8s-datascience-prd-ase1" target_label = "cluster" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { replacement = "kubernetes.default.svc:443" target_label = "__address__" } rule { source_labels = ["__meta_kubernetes_node_name"] regex = "(.+)" target_label = "__metrics_path__" replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" } } prometheus.scrape "cadvisor_kubernetes_nodes" { targets = discovery.relabel.cadvisor_kubernetes_nodes.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "cadvisor_kubernetes_nodes" scheme = "https" scrape_interval = "15s" authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } clustering { enabled = true } } // kubelet discovery.relabel "kubelet_kubernetes_nodes" { targets = discovery.kubernetes.kubernetes_nodes.targets rule { replacement = "k8s-datascience-prd-ase1" target_label = "cluster" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { target_label = "__address__" replacement = "kubernetes.default.svc:443" } rule { source_labels = ["__meta_kubernetes_node_name"] regex = "(.+)" target_label = "__metrics_path__" replacement = "/api/v1/nodes/$1/proxy/metrics" } } prometheus.scrape "kubelet_kubernetes_nodes" { targets = discovery.relabel.kubelet_kubernetes_nodes.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "kubelet_kubernetes_nodes" scheme = "https" scrape_interval = "15s" authorization { type = "Bearer" credentials_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" } tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } clustering { enabled = true } } // prometheus_kubernetes_endpoints discovery.kubernetes "kubernetes_endpoints" { role = "endpoints" } discovery.relabel "prometheus_kubernetes_endpoints" { targets = discovery.kubernetes.kubernetes_endpoints.targets rule { source_labels = ["__meta_kubernetes_pod_container_init"] regex = "true" action = "drop" } rule { source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_port"] target_label = "__meta_kubernetes_pod_container_port_number" action = "keepequal" } rule { source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"] regex = "true" action = "keep" } rule { source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"] regex = "(.+)" target_label = "__metrics_path__" } rule { source_labels = ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"] regex = "([^:]+)(?::\\d+)?;(\\d+)" target_label = "__address__" replacement = "$1:$2" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { regex = "__meta_kubernetes_service_label_(.+)" action = "labelmap" } rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "kubernetes_namespace" } rule { source_labels = ["__meta_kubernetes_endpoint_node_name"] target_label = "kubernetes_node" } rule { source_labels = ["__meta_kubernetes_service_name"] target_label = "kubernetes_service_name" } rule { source_labels = ["__meta_kubernetes_endpoints_name"] target_label = "kubernetes_endpoints_name" } rule { source_labels = ["__meta_kubernetes_pod_name"] target_label = "kubernetes_pod_name" } rule { target_label = "cluster" replacement = "k8s-datascience-prd-ase1" } } prometheus.scrape "prometheus_kubernetes_endpoints" { targets = discovery.relabel.prometheus_kubernetes_endpoints.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "prometheus_kubernetes_endpoints" clustering { enabled = true } } // prometheus_kubernetes_pods discovery.kubernetes "kubernetes_pods" { role = "pod" } discovery.relabel "prometheus_kubernetes_pods" { targets = discovery.kubernetes.kubernetes_pods.targets rule { source_labels = ["__meta_kubernetes_pod_container_init"] regex = "true" action = "drop" } rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port"] target_label = "__meta_kubernetes_pod_container_port_number" action = "keepequal" } rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"] regex = "true" action = "keep" } rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"] regex = "(.+)" target_label = "__metrics_path__" } rule { source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"] regex = "([^:]+)(?::\\d+)?;(\\d+)" target_label = "__address__" replacement = "$1:$2" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { regex = "__meta_kubernetes_pod_label_(.+)" action = "labelmap" } rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "kubernetes_namespace" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] target_label = "kubernetes_node" } rule { source_labels = ["__meta_kubernetes_pod_name"] target_label = "kubernetes_pod_name" } rule { target_label = "cluster" replacement = "k8s-datascience-prd-ase1" } } prometheus.scrape "prometheus_kubernetes_pods" { targets = discovery.relabel.prometheus_kubernetes_pods.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "prometheus_kubernetes_pods" clustering { enabled = true } } // jmx_kubernetes_pods discovery.relabel "jmx_kubernetes_pods" { targets = discovery.kubernetes.kubernetes_pods.targets rule { source_labels = ["__meta_kubernetes_pod_container_init"] regex = "true" action = "drop" } rule { source_labels = ["__meta_kubernetes_pod_annotation_jmx_io_port"] target_label = "__meta_kubernetes_pod_container_port_number" action = "keepequal" } rule { source_labels = ["__meta_kubernetes_pod_annotation_jmx_io_scrape"] regex = "true" action = "keep" } rule { source_labels = ["__meta_kubernetes_pod_annotation_jmx_io_path"] regex = "(.+)" target_label = "__metrics_path__" } rule { source_labels = ["__address__", "__meta_kubernetes_pod_annotation_jmx_io_port"] regex = "([^:]+)(?::\\d+)?;(\\d+)" target_label = "__address__" replacement = "$1:$2" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { regex = "__meta_kubernetes_pod_label_(.+)" action = "labelmap" } rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "kubernetes_namespace" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] target_label = "kubernetes_node" } rule { source_labels = ["__meta_kubernetes_pod_name"] target_label = "kubernetes_pod_name" } rule { target_label = "cluster" replacement = "k8s-datascience-prd-ase1" } } prometheus.scrape "jmx_kubernetes_pods" { targets = discovery.relabel.jmx_kubernetes_pods.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "jmx_kubernetes_pods" clustering { enabled = true } } // pm2_metrics discovery.relabel "pm2_kubernetes_pods" { targets = discovery.kubernetes.kubernetes_pods.targets rule { source_labels = ["__meta_kubernetes_pod_container_init"] regex = "true" action = "drop" } rule { source_labels = ["__address__"] regex = "(.*9209)" action = "keep" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { regex = "__meta_kubernetes_pod_label_(.+)" action = "labelmap" } rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "kubernetes_namespace" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] target_label = "kubernetes_node" } rule { source_labels = ["__meta_kubernetes_pod_name"] target_label = "kubernetes_pod_name" } rule { target_label = "cluster" replacement = "k8s-datascience-prd-ase1" } } prometheus.scrape "pm2_kubernetes_pods" { targets = discovery.relabel.pm2_kubernetes_pods.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "pm2_kubernetes_pods" clustering { enabled = true } } // scylladb_manager_exporter discovery.gce "scylladb_manager_a" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-a" port = 5090 } discovery.gce "scylladb_manager_b" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-b" port = 5090 } discovery.gce "scylladb_manager_c" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-c" port = 5090 } discovery.relabel "scylladb_manager_gce" { targets = concat( discovery.gce.scylladb_manager_a.targets, discovery.gce.scylladb_manager_b.targets, discovery.gce.scylladb_manager_c.targets, ) rule { source_labels = ["__meta_gce_label_component"] regex = "scyl" action = "keep" } rule { source_labels = ["__meta_gce_label_bu"] target_label = "bu" } rule { source_labels = ["__meta_gce_label_team"] target_label = "team" } rule { source_labels = ["__meta_gce_label_service"] target_label = "service" } rule { source_labels = ["__meta_gce_label_type"] target_label = "type" } rule { source_labels = ["__meta_gce_label_env"] target_label = "env" } rule { source_labels = ["__meta_gce_label_cluster"] target_label = "cluster" } rule { source_labels = ["__meta_gce_instance_name"] target_label = "name" } rule { source_labels = ["__meta_gce_label_priority"] target_label = "priority" } } prometheus.scrape "scylladb_manager_gce" { targets = discovery.relabel.scylladb_manager_gce.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "scylladb_manager_gce" clustering { enabled = true } } // scylladb_exporter discovery.gce "scylladb_a" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-a" port = 9180 } discovery.gce "scylladb_b" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-b" port = 9180 } discovery.gce "scylladb_c" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-c" port = 9180 } discovery.relabel "scylladb_gce" { targets = concat( discovery.gce.scylladb_a.targets, discovery.gce.scylladb_b.targets, discovery.gce.scylladb_c.targets, ) rule { source_labels = ["__meta_gce_label_component"] regex = "scyl" action = "keep" } rule { source_labels = ["__meta_gce_label_bu"] target_label = "bu" } rule { source_labels = ["__meta_gce_label_team"] target_label = "team" } rule { source_labels = ["__meta_gce_label_service"] target_label = "service" } rule { source_labels = ["__meta_gce_label_type"] target_label = "type" } rule { source_labels = ["__meta_gce_label_env"] target_label = "env" } rule { source_labels = ["__meta_gce_label_cluster"] target_label = "cluster" } rule { source_labels = ["__meta_gce_instance_name"] target_label = "name" } rule { source_labels = ["__meta_gce_label_priority"] target_label = "priority" } } prometheus.scrape "scylladb_gce" { targets = discovery.relabel.scylladb_gce.output forward_to = [prometheus.relabel.scylladb_gce.receiver] job_name = "scylladb_gce" clustering { enabled = true } } prometheus.relabel "scylladb_gce" { forward_to = [prometheus.remote_write.mimir.receiver] rule { source_labels = ["__name__", "scheduling_group_name"] regex = "(scylla_storage_proxy_coordinator_.*_bucket;)(atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache)" action = "drop" } rule { source_labels = ["version"] regex = "(.+)" target_label = "CPU" replacement = "cpu" } rule { source_labels = ["version"] regex = "(.+)" target_label = "CQL" replacement = "cql" } rule { source_labels = ["version"] regex = "(.+)" target_label = "OS" replacement = "os" } rule { source_labels = ["version"] regex = "(.+)" target_label = "IO" replacement = "io" } rule { source_labels = ["version"] regex = "(.+)" target_label = "Errors" replacement = "errors" } rule { regex = "help|exported_instance" action = "labeldrop" } rule { source_labels = ["version"] regex = "([0-9]+\\.[0-9]+)(\\.?[0-9]*).*" target_label = "svr" replacement = "$1$2" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_write_latency_summary;0.990*)" target_label = "__name__" replacement = "wlatencyp99" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_write_latency_summary;0.950*)" target_label = "__name__" replacement = "wlatencyp95" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_write_latency_summary;0.50*)" target_label = "__name__" replacement = "wlatencya" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_read_latency_summary;0.990*)" target_label = "__name__" replacement = "rlatencyp99" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_read_latency_summary;0.950*)" target_label = "__name__" replacement = "rlatencyp95" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_read_latency_summary;0.50*)" target_label = "__name__" replacement = "rlatencya" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_cas_write_latency_summary;0.950*)" target_label = "__name__" replacement = "caswlatencyp95" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_cas_write_latency_summary;0.990*)" target_label = "__name__" replacement = "caswlatencyp99" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_cas_write_latency_summary;0.50*)" target_label = "__name__" replacement = "caswlatencya" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_cas_read_latency_summary;0.950*)" target_label = "__name__" replacement = "casrlatencyp95" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_cas_read_latency_summary;0.990*)" target_label = "__name__" replacement = "casrlatencyp99" } rule { source_labels = ["__name__", "quantile"] regex = "(scylla_storage_proxy_coordinator_cas_read_latency_summary;0.50*)" target_label = "__name__" replacement = "casrlatencya" } rule { source_labels = ["__name__"] regex = "(.latency..?.?|cas.latency..?.?)" target_label = "by" replacement = "instance,shard" } rule { source_labels = ["__name__"] regex = "(scylla_storage_proxy_coordinator_read_timeouts|scylla_reactor_utilization|scylla_storage_proxy_coordinator_read_timeouts|scylla_storage_proxy_coordinator_read_unavailable|scylla_storage_proxy_coordinator_write_timeouts|scylla_storage_proxy_coordinator_write_unavailable|.latency..?.?)" target_label = "level" replacement = "2" } rule { source_labels = ["__name__"] regex = "(scylla_storage_proxy_coordinator_read_timeouts|scylla_reactor_utilization|scylla_storage_proxy_coordinator_read_timeouts|scylla_storage_proxy_coordinator_read_unavailable|scylla_storage_proxy_coordinator_write_timeouts|scylla_storage_proxy_coordinator_write_unavailable|.latency..?.?)" target_label = "dd" replacement = "2" } rule { source_labels = ["__name__"] regex = "(scylla_node_operation_mode)" target_label = "level" replacement = "1" } rule { source_labels = ["__name__"] regex = "(scylla_node_operation_mode)" target_label = "dd" replacement = "1" } rule { source_labels = ["scheduling_group_name"] regex = "(atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache)" target_label = "level" replacement = "" } rule { source_labels = ["scheduling_group_name"] regex = "(atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache)" target_label = "dd" replacement = "" } } // dragonfly mcache discovery.gce "dragonfly_mcache_a" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-a" port = 6379 } discovery.gce "dragonfly_mcache_b" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-b" port = 6379 } discovery.gce "dragonfly_mcache_c" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-c" port = 6379 } discovery.relabel "dragonfly_mcache_gce" { targets = concat( discovery.gce.dragonfly_mcache_a.targets, discovery.gce.dragonfly_mcache_b.targets, discovery.gce.dragonfly_mcache_c.targets,) rule { source_labels = ["__meta_gce_label_monitoring"] regex = "no" action = "drop" } rule { source_labels = ["__meta_gce_label_env"] regex = "(DR|perf)" action = "drop" } rule { source_labels = ["__meta_gce_label_pod"] regex = "(devops)" action = "drop" } rule { source_labels = ["__meta_gce_label_mcache"] regex = "(dragonfly)" action = "keep" } rule { source_labels = ["__meta_gce_label_name"] target_label = "name" } rule { source_labels = ["__meta_gce_label_mcache"] target_label = "mcache" } rule { source_labels = ["__meta_gce_label_bu"] target_label = "bu" } rule { source_labels = ["__meta_gce_label_team"] target_label = "team" } rule { source_labels = ["__meta_gce_label_env"] target_label = "env" } } prometheus.scrape "dragonfly_mcache_gce" { targets = discovery.relabel.dragonfly_mcache_gce.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "dragonfly_mcache_gce" clustering { enabled = true } } // zookeeper discovery.gce "zookeeper_a" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-a" port = 7000 } discovery.gce "zookeeper_b" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-b" port = 7000 } discovery.gce "zookeeper_c" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-c" port = 7000 } discovery.relabel "zookeeper_gce" { targets = concat( discovery.gce.zookeeper_a.targets, discovery.gce.zookeeper_b.targets, discovery.gce.zookeeper_c.targets, ) rule { source_labels = ["__meta_gce_label_service"] regex = "(comms-zookeeper)" action = "keep" } rule { target_label = "component" replacement = "zookeeper" } rule { target_label = "priority" replacement = "p0" } rule { target_label = "project" replacement = "meesho-datascience-prd-0622" } rule { source_labels = ["__meta_gce_label_bu"] target_label = "bu" } rule { source_labels = ["__meta_gce_label_team"] target_label = "team" } rule { source_labels = ["__meta_gce_label_service"] target_label = "app" } rule { source_labels = ["__meta_gce_label_type"] target_label = "type" } rule { source_labels = ["__meta_gce_label_env"] target_label = "env" } rule { source_labels = ["__meta_gce_instance_name"] target_label = "name" } } prometheus.scrape "zookeeper_gce" { targets = discovery.relabel.zookeeper_gce.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "zookeeper_gce" clustering { enabled = true } } // redis memcahe discovery.gce "redis_memcache_a" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-a" port = 9121 } discovery.gce "redis_memcache_b" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-b" port = 9121 } discovery.gce "redis_memcache_c" { project = "meesho-datascience-prd-0622" zone = "asia-southeast1-c" port = 9121 } discovery.relabel "redis_memcache" { targets = concat( discovery.gce.redis_memcache_a.targets, discovery.gce.redis_memcache_b.targets, discovery.gce.redis_memcache_c.targets, ) rule { source_labels = ["__meta_gce_label_monitoring"] regex = "no" action = "drop" } rule { source_labels = ["__meta_gce_label_env"] regex = "(DR|perf)" action = "drop" } rule { source_labels = ["__meta_gce_label_pod"] regex = "(devops)" action = "drop" } rule { source_labels = ["__meta_gce_label_mcache"] regex = "(redis-server)" action = "keep" } rule { source_labels = ["__meta_gce_label_name"] target_label = "name" } rule { source_labels = ["__meta_gce_label_mcache"] target_label = "mcache" } rule { source_labels = ["__meta_gce_label_bu"] target_label = "bu" } rule { source_labels = ["__meta_gce_label_team"] target_label = "team" } rule { source_labels = ["__meta_gce_label_env"] target_label = "env" } } prometheus.scrape "redis_memcache" { targets = discovery.relabel.redis_memcache.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "redis_memcache" scrape_timeout = "5s" clustering { enabled = true } } // model inference service discovery.relabel "model_inference_service_kubernetes_pods" { targets = discovery.kubernetes.kubernetes_pods.targets rule { source_labels = ["__address__"] regex = "(.*8002)" action = "keep" } rule { regex = "__meta_kubernetes_node_label_(.+)" action = "labelmap" } rule { regex = "__meta_kubernetes_pod_label_(.+)" action = "labelmap" } rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "kubernetes_namespace" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] target_label = "kubernetes_node" } rule { source_labels = ["__meta_kubernetes_pod_name"] target_label = "kubernetes_pod_name" } rule { target_label = "cluster" replacement = "k8s-datascience-prd-ase1" } } prometheus.scrape "model_inference_service_kubernetes_pods" { targets = discovery.relabel.model_inference_service_kubernetes_pods.output forward_to = [prometheus.remote_write.mimir.receiver] job_name = "model_inference_service_kubernetes_pods" clustering { enabled = true } } prometheus.exporter.self "default" {} prometheus.scrape "alloy_pods" { targets = prometheus.exporter.self.default.targets forward_to = [prometheus.remote_write.mimir.receiver] } // writers prometheus.remote_write "mimir" { endpoint { url = "http://mimir-nginx.mimir-distributed.svc.clusterset.local/api/v1/push" headers = {"X-Scope-OrgID" = "anonymous"} tls_config { insecure_skip_verify = true } queue_config { min_shards = 25 } } }