{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "8.0.0" } ], "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "mimir" ], "targetBlank": false, "title": "Mimir dashboards", "type": "dashboards" } ], "refresh": "10s", "rows": [ { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 1, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "CPU", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 2, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (workingset)", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 3, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Writes", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 4, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"distributor|ingester|mimir-write\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk writes", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 5, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"distributor|ingester|mimir-write\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk reads", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "percentunit" }, "overrides": [ ] }, "id": 6, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(distributor|ingester|mimir-write).*\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(distributor|ingester|mimir-write).*\"}\n)\n", "format": "time_series", "legendFormat": "{{persistentvolumeclaim}}", "legendLink": null } ], "title": "Disk space utilization", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 7, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "CPU", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 8, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (workingset)", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 9, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Reads", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 10, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "CPU", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 11, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (workingset)", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "bytes" }, "overrides": [ ] }, "id": 12, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Memory (go heap inuse)", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Backend", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 13, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk writes", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "Bps" }, "overrides": [ ] }, "id": 14, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "legendFormat": "{{pod}} - {{device}}", "legendLink": null } ], "title": "Disk reads", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "custom": { "fillOpacity": 0 }, "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "percentunit" }, "overrides": [ ] }, "id": 15, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}\n)\n", "format": "time_series", "legendFormat": "{{persistentvolumeclaim}}", "legendLink": null } ], "title": "Disk space utilization", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [ "mimir" ], "templating": { "list": [ { "current": { "text": "default", "value": "default" }, "hide": 0, "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": ".*", "current": { "text": "prod", "value": "prod" }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "cluster", "multi": false, "name": "cluster", "options": [ ], "query": "label_values(cortex_build_info, cluster)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": "prod", "value": "prod" }, "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "namespace", "multi": false, "name": "namespace", "options": [ ], "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "utc", "title": "Mimir / Overview resources", "uid": "a9b92d3c4d1af325d872a9e9a7083d71", "version": 0 }