4737 lines
211 KiB
JSON
4737 lines
211 KiB
JSON
{
|
||
"__requires": [
|
||
{
|
||
"id": "grafana",
|
||
"name": "Grafana",
|
||
"type": "grafana",
|
||
"version": "8.0.0"
|
||
}
|
||
],
|
||
"annotations": {
|
||
"list": [ ]
|
||
},
|
||
"editable": true,
|
||
"gnetId": null,
|
||
"graphTooltip": 1,
|
||
"hideControls": false,
|
||
"links": [
|
||
{
|
||
"asDropdown": true,
|
||
"icon": "external link",
|
||
"includeVars": true,
|
||
"keepTime": true,
|
||
"tags": [
|
||
"mimir"
|
||
],
|
||
"targetBlank": false,
|
||
"title": "Mimir dashboards",
|
||
"type": "dashboards"
|
||
}
|
||
],
|
||
"refresh": "10s",
|
||
"rows": [
|
||
{
|
||
"collapse": false,
|
||
"height": "175px",
|
||
"panels": [
|
||
{
|
||
"content": "<p>\n This dashboard shows health metrics for the read path.\n It is broken into sections for each service on the read path, and organized by the order in which the read request flows.\n <br/>\n Incoming queries travel from the gateway → query frontend → query scheduler → querier → ingester and/or store-gateway (depending on the time range of the query).\n <br/>\n For each service, there are 3 panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n</p>\n<p>\n The dashboard also shows metrics for the 4 optional caches that can be deployed:\n the query results cache, the metadata cache, the chunks cache, and the index cache.\n <br/>\n These panels will show “no data” if the caches are not deployed.\n</p>\n<p>\n Lastly, it also includes metrics for how the ingester and store-gateway interact with object storage.\n</p>\n",
|
||
"datasource": null,
|
||
"description": "",
|
||
"id": 1,
|
||
"mode": "markdown",
|
||
"span": 12,
|
||
"title": "",
|
||
"transparent": true,
|
||
"type": "text"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": false,
|
||
"title": "Reads dashboard description",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "100px",
|
||
"panels": [
|
||
{
|
||
"aliasColors": { },
|
||
"bars": false,
|
||
"dashLength": 10,
|
||
"dashes": false,
|
||
"datasource": "$datasource",
|
||
"description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the <tt>/prometheus</tt> API as\nwell as queries from the ruler.\n\n",
|
||
"fill": 1,
|
||
"format": "reqps",
|
||
"id": 2,
|
||
"legend": {
|
||
"avg": false,
|
||
"current": false,
|
||
"max": false,
|
||
"min": false,
|
||
"show": true,
|
||
"total": false,
|
||
"values": false
|
||
},
|
||
"lines": true,
|
||
"linewidth": 1,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"percentage": false,
|
||
"pointradius": 5,
|
||
"points": false,
|
||
"renderer": "flot",
|
||
"seriesOverrides": [ ],
|
||
"spaceLength": 10,
|
||
"span": 2,
|
||
"stack": false,
|
||
"steppedLine": false,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",\n route=~\"(prometheus|api_prom)_api_v1_query\"\n }[$__rate_interval]\n )\n or\n rate(\n cortex_prometheus_rule_evaluations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
|
||
"format": "time_series",
|
||
"instant": true,
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"thresholds": "70,80",
|
||
"timeFrom": null,
|
||
"timeShift": null,
|
||
"title": "Instant queries / sec",
|
||
"tooltip": {
|
||
"shared": false,
|
||
"sort": 0,
|
||
"value_type": "individual"
|
||
},
|
||
"type": "singlestat",
|
||
"xaxis": {
|
||
"buckets": null,
|
||
"mode": "time",
|
||
"name": null,
|
||
"show": true,
|
||
"values": [ ]
|
||
},
|
||
"yaxes": [
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"aliasColors": { },
|
||
"bars": false,
|
||
"dashLength": 10,
|
||
"dashes": false,
|
||
"datasource": "$datasource",
|
||
"description": "### Range queries per second\nRate of range queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
|
||
"fill": 1,
|
||
"format": "reqps",
|
||
"id": 3,
|
||
"legend": {
|
||
"avg": false,
|
||
"current": false,
|
||
"max": false,
|
||
"min": false,
|
||
"show": true,
|
||
"total": false,
|
||
"values": false
|
||
},
|
||
"lines": true,
|
||
"linewidth": 1,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"percentage": false,
|
||
"pointradius": 5,
|
||
"points": false,
|
||
"renderer": "flot",
|
||
"seriesOverrides": [ ],
|
||
"spaceLength": 10,
|
||
"span": 2,
|
||
"stack": false,
|
||
"steppedLine": false,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"instant": true,
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"thresholds": "70,80",
|
||
"timeFrom": null,
|
||
"timeShift": null,
|
||
"title": "Range queries / sec",
|
||
"tooltip": {
|
||
"shared": false,
|
||
"sort": 0,
|
||
"value_type": "individual"
|
||
},
|
||
"type": "singlestat",
|
||
"xaxis": {
|
||
"buckets": null,
|
||
"mode": "time",
|
||
"name": null,
|
||
"show": true,
|
||
"values": [ ]
|
||
},
|
||
"yaxes": [
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"aliasColors": { },
|
||
"bars": false,
|
||
"dashLength": 10,
|
||
"dashes": false,
|
||
"datasource": "$datasource",
|
||
"description": "### \"Label names\" queries per second\nRate of \"label names\" endpoint queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
|
||
"fill": 1,
|
||
"format": "reqps",
|
||
"id": 4,
|
||
"legend": {
|
||
"avg": false,
|
||
"current": false,
|
||
"max": false,
|
||
"min": false,
|
||
"show": true,
|
||
"total": false,
|
||
"values": false
|
||
},
|
||
"lines": true,
|
||
"linewidth": 1,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"percentage": false,
|
||
"pointradius": 5,
|
||
"points": false,
|
||
"renderer": "flot",
|
||
"seriesOverrides": [ ],
|
||
"spaceLength": 10,
|
||
"span": 2,
|
||
"stack": false,
|
||
"steppedLine": false,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"instant": true,
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"thresholds": "70,80",
|
||
"timeFrom": null,
|
||
"timeShift": null,
|
||
"title": "Label names queries / sec",
|
||
"tooltip": {
|
||
"shared": false,
|
||
"sort": 0,
|
||
"value_type": "individual"
|
||
},
|
||
"type": "singlestat",
|
||
"xaxis": {
|
||
"buckets": null,
|
||
"mode": "time",
|
||
"name": null,
|
||
"show": true,
|
||
"values": [ ]
|
||
},
|
||
"yaxes": [
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"aliasColors": { },
|
||
"bars": false,
|
||
"dashLength": 10,
|
||
"dashes": false,
|
||
"datasource": "$datasource",
|
||
"description": "### \"Label values\" queries per second\nRate of specific \"label values\" endpoint queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
|
||
"fill": 1,
|
||
"format": "reqps",
|
||
"id": 5,
|
||
"legend": {
|
||
"avg": false,
|
||
"current": false,
|
||
"max": false,
|
||
"min": false,
|
||
"show": true,
|
||
"total": false,
|
||
"values": false
|
||
},
|
||
"lines": true,
|
||
"linewidth": 1,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"percentage": false,
|
||
"pointradius": 5,
|
||
"points": false,
|
||
"renderer": "flot",
|
||
"seriesOverrides": [ ],
|
||
"spaceLength": 10,
|
||
"span": 2,
|
||
"stack": false,
|
||
"steppedLine": false,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"instant": true,
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"thresholds": "70,80",
|
||
"timeFrom": null,
|
||
"timeShift": null,
|
||
"title": "Label values queries / sec",
|
||
"tooltip": {
|
||
"shared": false,
|
||
"sort": 0,
|
||
"value_type": "individual"
|
||
},
|
||
"type": "singlestat",
|
||
"xaxis": {
|
||
"buckets": null,
|
||
"mode": "time",
|
||
"name": null,
|
||
"show": true,
|
||
"values": [ ]
|
||
},
|
||
"yaxes": [
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"aliasColors": { },
|
||
"bars": false,
|
||
"dashLength": 10,
|
||
"dashes": false,
|
||
"datasource": "$datasource",
|
||
"description": "### Series queries per second\nRate of series queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
|
||
"fill": 1,
|
||
"format": "reqps",
|
||
"id": 6,
|
||
"legend": {
|
||
"avg": false,
|
||
"current": false,
|
||
"max": false,
|
||
"min": false,
|
||
"show": true,
|
||
"total": false,
|
||
"values": false
|
||
},
|
||
"lines": true,
|
||
"linewidth": 1,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"percentage": false,
|
||
"pointradius": 5,
|
||
"points": false,
|
||
"renderer": "flot",
|
||
"seriesOverrides": [ ],
|
||
"spaceLength": 10,
|
||
"span": 2,
|
||
"stack": false,
|
||
"steppedLine": false,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"instant": true,
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"thresholds": "70,80",
|
||
"timeFrom": null,
|
||
"timeShift": null,
|
||
"title": "Series queries / sec",
|
||
"tooltip": {
|
||
"shared": false,
|
||
"sort": 0,
|
||
"value_type": "individual"
|
||
},
|
||
"type": "singlestat",
|
||
"xaxis": {
|
||
"buckets": null,
|
||
"mode": "time",
|
||
"name": null,
|
||
"show": true,
|
||
"values": [ ]
|
||
},
|
||
"yaxes": [
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": false,
|
||
"title": "Headlines",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "1xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EAB839",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "2xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "3xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#6ED0E0",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "4xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EF843C",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "5xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "OK"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "cancel"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#A9A9A9",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "error"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 7,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{status}}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 8,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 0,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "s"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 9,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "hidden",
|
||
"showLegend": false
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"exemplar": true,
|
||
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
|
||
"format": "time_series",
|
||
"legendFormat": "",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Per pod p99 latency",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Query-frontend",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Requests / sec\n<p>\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n</p>\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "1xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EAB839",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "2xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "3xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#6ED0E0",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "4xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EF843C",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "5xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "OK"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "cancel"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#A9A9A9",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "error"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 10,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{status}}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Latency (Time in Queue)\n<p>\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n</p>\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 11,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency (Time in Queue)",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Queue length\n<p>\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n</p>\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 0,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "queries"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 12,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "hidden",
|
||
"showLegend": false
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"exemplar": true,
|
||
"expr": "sum(min_over_time(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Queue length",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Queue length",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Query-scheduler",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### 99th Percentile Latency by Queue Dimension\n<p>\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n</p>\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 13,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "99th Percentile Latency by Queue Dimension",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### 50th Percentile Latency by Queue Dimension\n<p>\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n</p>\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 14,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "50th Percentile Latency by Queue Dimension",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Average Latency by Queue Dimension\n<p>\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n</p>\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 15,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
|
||
"format": "time_series",
|
||
"legendFormat": "Average: {{ additional_queue_dimensions }}",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Average Latency by Queue Dimension",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Query-scheduler Latency (Time in Queue) Breakout by Additional Queue Dimensions",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ops"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 16,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 6,
|
||
"targets": [
|
||
{
|
||
"expr": "sum (\n rate(thanos_memcached_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "Requests/s",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 17,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 6,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Cache – query results",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "1xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EAB839",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "2xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "3xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#6ED0E0",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "4xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EF843C",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "5xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "OK"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "cancel"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#A9A9A9",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "error"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 18,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{status}}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 19,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 0,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "s"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 20,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "hidden",
|
||
"showLegend": false
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"exemplar": true,
|
||
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
|
||
"format": "time_series",
|
||
"legendFormat": "",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Per pod p99 latency",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Querier",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "1xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EAB839",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "2xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "3xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#6ED0E0",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "4xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EF843C",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "5xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "OK"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "cancel"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#A9A9A9",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "error"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 21,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{status}}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 22,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"})",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 0,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "s"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 23,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "hidden",
|
||
"showLegend": false
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"exemplar": true,
|
||
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"}[$__rate_interval])))",
|
||
"format": "time_series",
|
||
"legendFormat": "",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Per pod p99 latency",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Ingester",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "1xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EAB839",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "2xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "3xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#6ED0E0",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "4xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EF843C",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "5xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "OK"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "cancel"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#A9A9A9",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "error"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 24,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{status}}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 25,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 0,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "s"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 26,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "hidden",
|
||
"showLegend": false
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"exemplar": true,
|
||
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))",
|
||
"format": "time_series",
|
||
"legendFormat": "",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Per pod p99 latency",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Store-gateway",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Replicas\nThe maximum, and current number of querier replicas.\nPlease note that the current number of replicas can still show 1 replica even when scaled to 0.\nSince HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "short"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byRegexp",
|
||
"options": "/Max .+/"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "custom.fillOpacity",
|
||
"value": 0
|
||
},
|
||
{
|
||
"id": "custom.lineStyle",
|
||
"value": {
|
||
"fill": "dash"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byRegexp",
|
||
"options": "/Current .+/"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "custom.fillOpacity",
|
||
"value": 0
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byRegexp",
|
||
"options": "/Min .+/"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "custom.fillOpacity",
|
||
"value": 0
|
||
},
|
||
{
|
||
"id": "custom.lineStyle",
|
||
"value": {
|
||
"fill": "dash"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 27,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "Max {{ scaletargetref_name }}",
|
||
"legendLink": null
|
||
},
|
||
{
|
||
"expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active.\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "Current {{ scaletargetref_name }}",
|
||
"legendLink": null
|
||
},
|
||
{
|
||
"expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "Min {{ scaletargetref_name }}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Replicas",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Scaling metric (desired replicas)\nThis panel shows the result scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints which are applied later.\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "short"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 28,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left\n label_replace(label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ), \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\")\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{ scaler }}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Scaling metric (desired replicas)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "short"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 29,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{scaler}} failures",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Autoscaler failures rate",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Querier - autoscaling",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "1xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EAB839",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "2xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "3xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#6ED0E0",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "4xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#EF843C",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "5xx"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "OK"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "cancel"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#A9A9A9",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "error"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#E24D42",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "color",
|
||
"value": {
|
||
"fixedColor": "#7EB26D",
|
||
"mode": "fixed"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"id": 30,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 6,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{status}}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 31,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 6,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Store-gateway – key-value store for store-gateways ring",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ops"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 32,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (\n # Backwards compatibility\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n or ignoring(backend)\n rate(\n thanos_cache_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 33,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency (getmulti)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"description": "### Hit ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because the store-gateway by default has an\nin-memory blocks index cache.\n\n",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "percentunit"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 34,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{item_type}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Hit ratio",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Memcached – block index cache (store-gateway accesses)",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ops"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 35,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 36,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency (getmulti)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "percentunit"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 37,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "items",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Hit ratio",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Memcached – chunks cache (store-gateway accesses)",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ops"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 38,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 39,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency (getmulti)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "percentunit"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 40,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "items",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Hit ratio",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Memcached – metadata cache (store-gateway accesses)",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ops"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 41,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Requests / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"noValue": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 42,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"intervalFactor": 2,
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency (getmulti)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "percentunit"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 43,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 4,
|
||
"targets": [
|
||
{
|
||
"expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
|
||
"format": "time_series",
|
||
"legendFormat": "items",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Hit ratio",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Memcached – metadata cache (querier accesses)",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 44,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Operations / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"max": 1,
|
||
"min": 0,
|
||
"noValue": "0",
|
||
"unit": "percentunit"
|
||
}
|
||
},
|
||
"id": 45,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) >= 0",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Error rate",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 46,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Attributes",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 47,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Exists",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Blocks object store (store-gateway accesses)",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 48,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Get",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 49,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: GetRange",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 50,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Upload",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 51,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Delete",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 100,
|
||
"lineWidth": 0,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "normal"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 52,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Operations / sec",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"max": 1,
|
||
"min": 0,
|
||
"noValue": "0",
|
||
"unit": "percentunit"
|
||
}
|
||
},
|
||
"id": 53,
|
||
"links": [ ],
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) >= 0",
|
||
"format": "time_series",
|
||
"legendFormat": "{{operation}}",
|
||
"legendLink": null
|
||
}
|
||
],
|
||
"title": "Error rate",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 54,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Attributes",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 55,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Exists",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "Blocks object store (querier accesses)",
|
||
"titleSize": "h6"
|
||
},
|
||
{
|
||
"collapse": false,
|
||
"height": "250px",
|
||
"panels": [
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 56,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Get",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 57,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: GetRange",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 58,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Upload",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"datasource": "$datasource",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"fillOpacity": 1,
|
||
"lineWidth": 1,
|
||
"pointSize": 5,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"stacking": {
|
||
"group": "A",
|
||
"mode": "none"
|
||
}
|
||
},
|
||
"min": 0,
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [ ]
|
||
},
|
||
"unit": "ms"
|
||
},
|
||
"overrides": [ ]
|
||
},
|
||
"id": 59,
|
||
"links": [ ],
|
||
"nullPointMode": "null as zero",
|
||
"options": {
|
||
"legend": {
|
||
"showLegend": true
|
||
},
|
||
"tooltip": {
|
||
"mode": "single",
|
||
"sort": "none"
|
||
}
|
||
},
|
||
"span": 3,
|
||
"targets": [
|
||
{
|
||
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "99th Percentile",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
|
||
"format": "time_series",
|
||
"legendFormat": "50th Percentile",
|
||
"refId": "B"
|
||
},
|
||
{
|
||
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))",
|
||
"format": "time_series",
|
||
"legendFormat": "Average",
|
||
"refId": "C"
|
||
}
|
||
],
|
||
"title": "Latency of op: Delete",
|
||
"type": "timeseries",
|
||
"yaxes": [
|
||
{
|
||
"format": "ms",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": 0,
|
||
"show": true
|
||
},
|
||
{
|
||
"format": "short",
|
||
"label": null,
|
||
"logBase": 1,
|
||
"max": null,
|
||
"min": null,
|
||
"show": false
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"repeat": null,
|
||
"repeatIteration": null,
|
||
"repeatRowId": null,
|
||
"showTitle": true,
|
||
"title": "",
|
||
"titleSize": "h6"
|
||
}
|
||
],
|
||
"schemaVersion": 14,
|
||
"style": "dark",
|
||
"tags": [
|
||
"mimir"
|
||
],
|
||
"templating": {
|
||
"list": [
|
||
{
|
||
"current": {
|
||
"text": "default",
|
||
"value": "default"
|
||
},
|
||
"hide": 0,
|
||
"label": "Data source",
|
||
"name": "datasource",
|
||
"options": [ ],
|
||
"query": "prometheus",
|
||
"refresh": 1,
|
||
"regex": "",
|
||
"type": "datasource"
|
||
},
|
||
{
|
||
"allValue": ".+",
|
||
"current": {
|
||
"selected": true,
|
||
"text": "All",
|
||
"value": "$__all"
|
||
},
|
||
"datasource": "$datasource",
|
||
"hide": 0,
|
||
"includeAll": true,
|
||
"label": "cluster",
|
||
"multi": true,
|
||
"name": "cluster",
|
||
"options": [ ],
|
||
"query": "label_values(cortex_build_info, cluster)",
|
||
"refresh": 1,
|
||
"regex": "",
|
||
"sort": 1,
|
||
"tagValuesQuery": "",
|
||
"tags": [ ],
|
||
"tagsQuery": "",
|
||
"type": "query",
|
||
"useTags": false
|
||
},
|
||
{
|
||
"allValue": ".+",
|
||
"current": {
|
||
"selected": true,
|
||
"text": "All",
|
||
"value": "$__all"
|
||
},
|
||
"datasource": "$datasource",
|
||
"hide": 0,
|
||
"includeAll": true,
|
||
"label": "namespace",
|
||
"multi": true,
|
||
"name": "namespace",
|
||
"options": [ ],
|
||
"query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
|
||
"refresh": 1,
|
||
"regex": "",
|
||
"sort": 1,
|
||
"tagValuesQuery": "",
|
||
"tags": [ ],
|
||
"tagsQuery": "",
|
||
"type": "query",
|
||
"useTags": false
|
||
}
|
||
]
|
||
},
|
||
"time": {
|
||
"from": "now-1h",
|
||
"to": "now"
|
||
},
|
||
"timepicker": {
|
||
"refresh_intervals": [
|
||
"5s",
|
||
"10s",
|
||
"30s",
|
||
"1m",
|
||
"5m",
|
||
"15m",
|
||
"30m",
|
||
"1h",
|
||
"2h",
|
||
"1d"
|
||
],
|
||
"time_options": [
|
||
"5m",
|
||
"15m",
|
||
"1h",
|
||
"6h",
|
||
"12h",
|
||
"24h",
|
||
"2d",
|
||
"7d",
|
||
"30d"
|
||
]
|
||
},
|
||
"timezone": "utc",
|
||
"title": "Mimir / Reads",
|
||
"uid": "e327503188913dc38ad571c647eef643",
|
||
"version": 0
|
||
} |