\n This dashboard shows various health metrics for the write path.\n It is broken into sections for each service on the write path,\n and organized by the order in which the write request flows.\n
\n Incoming metrics data travels from the gateway → distributor → ingester.\n
\n For each service, there are 3 panels showing\n (1) requests per second to that service,\n (2) average, median, and p99 latency of requests to that service, and\n (3) p99 latency of requests to each instance of that service.\n
\n It also includes metrics for the key-value (KV) stores used to manage\n the high-availability tracker and the ingesters.\n
\n", "datasource": null, "description": "", "id": 1, "mode": "markdown", "span": 12, "title": "", "transparent": true, "type": "text" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Writes dashboard description", "titleSize": "h6" }, { "collapse": false, "height": "100px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "format": "short", "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 2, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})", "format": "time_series", "instant": true, "refId": "A" } ], "thresholds": "70,80", "timeFrom": null, "timeShift": null, "title": "Samples / sec", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "### Exemplars / sec\nThe total number of received exemplars by the distributors, excluding rejected and deduped exemplars, but not necessarily ingested by the ingesters.\n\n", "fill": 1, "format": "short", "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 2, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})", "format": "time_series", "instant": true, "refId": "A" } ], "thresholds": "70,80", "timeFrom": null, "timeShift": null, "title": "Exemplars / sec", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "### In-memory series\nThe number of series not yet flushed to object storage that are held in ingester memory.\nWith classic storage we the sum of series from all ingesters is divided by the replication factor.\nWith ingest storage we take the maximum series of each ingest partition.\n\n", "fill": 1, "format": "short", "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 2, "stack": false, "steppedLine": false, "targets": [ { "expr": "( # Classic storage\n sum by (cluster, namespace, ) (cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})\n / on (cluster, namespace) group_left()\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\nor\n( # Ingest storage\n sum by (cluster, namespace, ) (\n max by (ingester_id, cluster, namespace, ) (\n label_replace(\n cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"},\n \"ingester_id\", \"$1\", \"pod\", \".*-([0-9]+)$\"\n )\n )\n )\n)\n", "format": "time_series", "instant": true, "refId": "A" } ], "thresholds": "70,80", "timeFrom": null, "timeShift": null, "title": "In-memory series", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "### Exemplars in ingesters\nNumber of TSDB exemplars currently in ingesters' storage.\nWith classic storage we the sum of exemplars from all ingesters is divided by the replication factor.\nWith ingest storage we take the maximum exemplars of each ingest partition.\n\n", "fill": 1, "format": "short", "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 2, "stack": false, "steppedLine": false, "targets": [ { "expr": "( # Classic storage\n sum by (cluster, namespace, ) (cortex_ingester_tsdb_exemplar_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})\n / on (cluster, namespace) group_left()\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\nor\n( # Ingest storage\n sum by (cluster, namespace, ) (\n max by (ingester_id, cluster, namespace, ) (\n label_replace(\n cortex_ingester_tsdb_exemplar_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"},\n \"ingester_id\", \"$1\", \"pod\", \".*-([0-9]+)$\"\n )\n )\n )\n)\n", "format": "time_series", "instant": true, "refId": "A" } ], "thresholds": "70,80", "timeFrom": null, "timeShift": null, "title": "Exemplars in ingesters", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "format": "short", "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 2, "stack": false, "steppedLine": false, "targets": [ { "expr": "count(count by(user) (cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}))", "format": "time_series", "instant": true, "refId": "A" } ], "thresholds": "70,80", "timeFrom": null, "timeShift": null, "title": "Tenants", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Headlines", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to distributor.\nRejected requests are requests that distributor fails to handle because of distributor instance limits.\nWhen distributor is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen distributor is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "1xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#EAB839", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "2xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "3xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#6ED0E0", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "4xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#EF843C", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "5xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "OK" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "cancel" }, "properties": [ { "id": "color", "value": { "fixedColor": "#A9A9A9", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] } ] }, "id": 7, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "legendFormat": "{{status}}", "refId": "A" } ], "title": "Requests / sec", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 8, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3", "format": "time_series", "legendFormat": "99th percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3", "format": "time_series", "legendFormat": "50th percentile", "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "s" }, "overrides": [ ] }, "id": 9, "links": [ ], "options": { "legend": { "displayMode": "hidden", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "span": 4, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval])))", "format": "time_series", "legendFormat": "", "legendLink": null } ], "title": "Per pod p99 latency", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Distributor", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to ingester.\nRejected requests are requests that ingester fails to handle because of ingester instance limits (ingester-max-inflight-push-requests, ingester-max-inflight-push-requests-bytes, ingester-max-ingestion-rate).\nWhen ingester is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen ingester is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "1xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#EAB839", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "2xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "3xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#6ED0E0", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "4xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#EF843C", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "5xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "OK" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "cancel" }, "properties": [ { "id": "color", "value": { "fixedColor": "#A9A9A9", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] } ] }, "id": 10, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "legendFormat": "{{status}}", "refId": "A" } ], "title": "Requests / sec", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 11, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3", "format": "time_series", "legendFormat": "99th percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3", "format": "time_series", "legendFormat": "50th percentile", "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "s" }, "overrides": [ ] }, "id": 12, "links": [ ], "options": { "legend": { "displayMode": "hidden", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "span": 4, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))", "format": "time_series", "legendFormat": "", "legendLink": null } ], "title": "Per pod p99 latency", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Ingester", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "description": "### Replicas\nThe maximum and current number of distributor replicas.