{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "8.0.0" } ], "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "mimir" ], "targetBlank": false, "title": "Mimir dashboards", "type": "dashboards" } ], "refresh": "10s", "rows": [ { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "completed" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "started" }, "properties": [ { "id": "color", "value": { "fixedColor": "#34CCEB", "mode": "fixed" } } ] } ] }, "id": 1, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "started", "legendLink": null }, { "expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "completed", "legendLink": null }, { "expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "failed", "legendLink": null } ], "title": "Per-instance runs / sec", "type": "timeseries" }, { "datasource": "$datasource", "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\n\n", "fieldConfig": { "defaults": { "max": 1, "noValue": 1, "unit": "percentunit" } }, "id": 2, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "(\n cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}\n)\n/\ncortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} > 0\n", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null } ], "title": "Tenants compaction progress", "type": "timeseries" }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "### Longest time since last successful run\nDisplays the amount of time since the most recent successful execution\nof the compactor.\nThe value shown will be for the compactor replica that has the longest time since its\nlast successful run.\nThe table to the right shows a summary for all compactor replicas.\n\nIf there is no time value, one of the following messages might appear:\n\n- If you see \"No compactor data\" in this panel, that means that no compactors are active yet.\n\n- If you see \"No successful runs\" in this panel, that means that compactors are active, but none\n of them were successfully executed yet.\n\nThese might be expected - for example, if you just recently restarted your compactors,\nthey might not have had a chance to complete their first compaction run.\nHowever, if these messages persist, you should check the health of your compactors.\n\n", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "noValue": "No compactor data", "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "Last run" }, "properties": [ { "id": "custom.width", "value": 74 }, { "id": "mappings", "value": [ { "options": { "from": "-Infinity", "result": { "color": "text", "text": "No successful runs since startup yet" }, "to": 0 }, "type": "range" } ] }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 7200 }, { "color": "orange", "value": 21600 }, { "color": "red", "value": 43200 } ] } } ] } ] }, "fill": 1, "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "options": { "reduceOptions": { "calcs": [ "first" ], "fields": "/^Last run$/", "values": false }, "textMode": "value" }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 3, "stack": false, "steppedLine": false, "targets": [ { "expr": "max by(pod)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n", "format": "table", "instant": true, "interval": "", "intervalFactor": null, "legendFormat": "", "legendLink": null, "step": null } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Longest time since last successful run", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "organize", "options": { "renameByName": { "Value": "Last run", "pod": "Compactor" } } }, { "id": "sortBy", "options": { "sort": [ { "desc": true, "field": "Last run" } ] } } ], "type": "stat", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "datasource": "$datasource", "description": "### Last successful run per-compactor replica\nDisplays the compactor replicas, and for each, shows how long it has been since\nits last successful compaction run.\n\nThe value in the status column is based on how long it has been since the last successful compaction.\n\n- Okay: less than 2 hours\n- Delayed: more than 2 hours\n- Late: more than 6 hours\n- Very late: more than 12 hours\n\nIf the status of any compactor replicas are *Late* or *Very late*, check their health.\n\n", "fieldConfig": { "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" }, { "id": "mappings", "value": [ { "options": { "from": "-Infinity", "result": { "color": "transparent", "text": "N/A" }, "to": 0 }, "type": "range" }, { "options": { "from": 0, "result": { "color": "green", "text": "Ok" }, "to": 7200 }, "type": "range" }, { "options": { "from": 7200, "result": { "color": "yellow", "text": "Delayed" }, "to": 21600 }, "type": "range" }, { "options": { "from": 21600, "result": { "color": "orange", "text": "Late" }, "to": 43200 }, "type": "range" }, { "options": { "from": 43200, "result": { "color": "red", "text": "Very late" }, "to": "Infinity" }, "type": "range" }, { "options": { "match": "null+nan", "result": { "color": "transparent", "text": "Unknown" } }, "type": "special" } ] }, { "id": "custom.width", "value": 86 }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Last run" }, "properties": [ { "id": "unit", "value": "s" }, { "id": "custom.width", "value": 74 }, { "id": "mappings", "value": [ { "options": { "from": "-Infinity", "result": { "text": "Never" }, "to": 0 }, "type": "range" } ] } ] } ] }, "id": 4, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "max by(pod)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n", "format": "table", "instant": true, "legendFormat": "Last run", "legendLink": null } ], "title": "Last successful run per-compactor replica", "transformations": [ { "id": "organize", "options": { "renameByName": { "Value": "Last run", "pod": "Compactor" } } }, { "id": "sortBy", "options": { "sort": [ { "desc": true, "field": "Last run" } ] } }, { "id": "calculateField", "options": { "alias": "One", "binary": { "left": "Last run", "operator": "/", "right": "Last run" }, "mode": "binary", "replaceFields": false } }, { "id": "calculateField", "options": { "alias": "Status", "binary": { "left": "Last run", "operator": "*", "right": "One" }, "mode": "binary", "replaceFields": false } }, { "id": "filterFieldsByName", "options": { "include": { "names": [ "Compactor", "Last run", "Status" ] } } } ], "type": "table" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Summary", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "description": "### Estimated Compaction Jobs\nEstimated number of compaction jobs based on latest version of bucket index. Ingesters upload new blocks every 2 hours (shortly after 01:00 UTC, 03:00 UTC, 05:00 UTC, etc.),\nand compactors should process all of them within 2h interval. If this graph regularly goes to zero (or close to zero) in 2 hour intervals, then compaction works as designed.\n\nMetric with number of compaction jobs is computed from blocks in bucket index, which is updated regularly. Metric doesn't change between bucket index updates, even if\nthere were compaction jobs finished in this time. When computing compaction jobs, only jobs that can be executed at given moment are counted. There can be more\njobs, but if they are blocked, they are not counted in the metric. For example if there is a split compaction job pending for some time range, no merge job\ncovering the same time range can run. In this case only split compaction job is counted toward the metric, but merge job isn't.\n\nIn other words, computed number of compaction jobs is the minimum number of compaction jobs based on latest version of bucket index.\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 5, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum(cortex_bucket_index_estimated_compaction_jobs{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}) and (sum(rate(cortex_bucket_index_estimated_compaction_jobs_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) == 0)", "format": "time_series", "legendFormat": "Jobs", "legendLink": null } ], "title": "Estimated Compaction Jobs", "type": "timeseries" }, { "datasource": "$datasource", "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ops" }, "overrides": [ ] }, "id": 6, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "compactions", "legendLink": null } ], "title": "TSDB compactions / sec", "type": "timeseries" }, { "datasource": "$datasource", "description": "### TSDB compaction duration\nDisplay the amount of time that it has taken to run a single TSDB compaction.\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 7, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "TSDB compaction duration", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 8, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))", "format": "time_series", "legendFormat": "avg", "legendLink": null } ], "title": "Average blocks / tenant", "type": "timeseries" }, { "datasource": "$datasource", "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "short" }, "overrides": [ ] }, "id": 9, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))", "format": "time_series", "legendFormat": "{{user}}", "legendLink": null } ], "title": "Tenants with largest number of blocks", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ops" }, "overrides": [ ] }, "id": 10, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n", "format": "time_series", "legendFormat": "blocks", "legendLink": null } ], "title": "Blocks marked for deletion / sec", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "successful" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] } ] }, "id": 11, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n", "format": "time_series", "legendFormat": "successful", "legendLink": null }, { "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n", "format": "time_series", "legendFormat": "failed", "legendLink": null } ], "title": "Blocks deletions / sec", "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Garbage collector", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "successful" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] } ] }, "id": 12, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n", "format": "time_series", "legendFormat": "successful", "legendLink": null }, { "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n", "format": "time_series", "legendFormat": "failed", "legendLink": null } ], "title": "Metadata syncs / sec", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 13, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Metadata sync duration", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Metadata sync", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "reqps" }, "overrides": [ ] }, "id": 14, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null } ], "title": "Operations / sec", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "0", "unit": "percentunit" } }, "id": 15, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) >= 0", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null } ], "title": "Error rate", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 16, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency of op: Attributes", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 17, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency of op: Exists", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Object Store", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 18, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency of op: Get", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 19, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency of op: GetRange", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 20, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency of op: Upload", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 21, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency of op: Delete", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 100, "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "1xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#EAB839", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "2xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "3xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#6ED0E0", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "4xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#EF843C", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "5xx" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "OK" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "cancel" }, "properties": [ { "id": "color", "value": { "fixedColor": "#A9A9A9", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E24D42", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] } ] }, "id": 22, "links": [ ], "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "legendFormat": "{{status}}", "refId": "A" } ], "title": "Requests / sec", "type": "timeseries" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 1, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ ] }, "unit": "ms" }, "overrides": [ ] }, "id": 23, "links": [ ], "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "span": 6, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" } ], "title": "Latency", "type": "timeseries", "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Key-value store for compactors ring", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [ "mimir" ], "templating": { "list": [ { "current": { "text": "default", "value": "default" }, "hide": 0, "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": ".+", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "cluster", "multi": true, "name": "cluster", "options": [ ], "query": "label_values(cortex_build_info, cluster)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".+", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "namespace", "multi": true, "name": "namespace", "options": [ ], "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "utc", "title": "Mimir / Compactor", "uid": "1b3443aea86db629e6efdb7d05c53823", "version": 0 }