{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "8.0.0" } ], "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "mimir" ], "targetBlank": false, "title": "Mimir dashboards", "type": "dashboards" } ], "refresh": "10s", "rows": [ { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 1, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Receive bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 2, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Transmit bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 3, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"})", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"})", "format": "time_series", "legendFormat": "highest", "legendLink": null } ], "title": "Inflight requests (per pod)", "type": "timeseries" }, { "datasource": "$datasource", "description": "### Ingress TCP connections (per pod)\nThe number of ingress TCP connections (HTTP and gRPC protocol).\n", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 4, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}))", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}))", "format": "time_series", "legendFormat": "highest", "legendLink": null }, { "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Ingress TCP connections (per pod)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Summary", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 5, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Receive bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 6, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Transmit bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 7, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"})", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"})", "format": "time_series", "legendFormat": "highest", "legendLink": null } ], "title": "Inflight requests (per pod)", "type": "timeseries" }, { "datasource": "$datasource", "description": "### Ingress TCP connections (per pod)\nThe number of ingress TCP connections (HTTP and gRPC protocol).\n", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 8, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}))", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}))", "format": "time_series", "legendFormat": "highest", "legendLink": null }, { "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Ingress TCP connections (per pod)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Ruler-query-frontend", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 9, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Receive bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 10, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Transmit bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 11, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"})", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"})", "format": "time_series", "legendFormat": "highest", "legendLink": null } ], "title": "Inflight requests (per pod)", "type": "timeseries" }, { "datasource": "$datasource", "description": "### Ingress TCP connections (per pod)\nThe number of ingress TCP connections (HTTP and gRPC protocol).\n", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 12, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}))", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}))", "format": "time_series", "legendFormat": "highest", "legendLink": null }, { "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Ingress TCP connections (per pod)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Ruler-query-scheduler", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 13, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Receive bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 14, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Transmit bandwidth", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 15, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"})", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"})", "format": "time_series", "legendFormat": "highest", "legendLink": null } ], "title": "Inflight requests (per pod)", "type": "timeseries" }, { "datasource": "$datasource", "description": "### Ingress TCP connections (per pod)\nThe number of ingress TCP connections (HTTP and gRPC protocol).\n", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 16, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}))", "format": "time_series", "legendFormat": "avg", "legendLink": null }, { "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}))", "format": "time_series", "legendFormat": "highest", "legendLink": null }, { "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"})", "format": "time_series", "legendFormat": "limit", "legendLink": null } ], "title": "Ingress TCP connections (per pod)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Ruler-querier", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [ "mimir" ], "templating": { "list": [ { "current": { "text": "default", "value": "default" }, "hide": 0, "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": ".*", "current": { "text": "prod", "value": "prod" }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "cluster", "multi": false, "name": "cluster", "options": [ ], "query": "label_values(cortex_build_info, cluster)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": "prod", "value": "prod" }, "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "namespace", "multi": false, "name": "namespace", "options": [ ], "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "utc", "title": "Mimir / Remote ruler reads networking", "uid": "9e8cfff65f91632f8a25981c6fe44bc9", "version": 0 }