一、Apiserver监控大盘
{
"name": "Kubernetes / API Server",
"tags": "Categraf",
"configs": {
"version": "2.0.0",
"links": [],
"var": [],
"panels": [
{
"targets": [
{
"refId": "A",
"expr": "up{job=\"apiserver\"}",
"legend": "{{ instance }}"
}
],
"name": "API Server - Health Status",
"links": [],
"description": "apiserver的实例健康状态，0表示down，1表示up",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value"
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "UP"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#f80202",
"text": "DOWN"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 0,
"i": "98f46bc1-c078-40f2-915c-f0836957bf2f",
"isResizable": true
},
"id": "98f46bc1-c078-40f2-915c-f0836957bf2f"
},
{
"targets": [
{
"refId": "A",
"expr": "apiserver_requested_deprecated_apis{job=\"apiserver\"}",
"legend": ""
}
],
"name": "Deprecated Kubernetes Resources",
"links": [],
"description": "当前版本apiserver使用，未来版本中要移除的资源",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "labelsOfSeriesToRows",
"columns": [
"group",
"version",
"resource",
"removed_release"
],
"sortOrder": "ascend"
},
"options": {
"standardOptions": {}
},
"overrides": [
{}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 0,
"i": "73beb13a-bd10-4a68-bb9e-5b9ab63da154",
"isResizable": true
},
"id": "73beb13a-bd10-4a68-bb9e-5b9ab63da154"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by (instance,code) (rate(apiserver_request_total{job=\"apiserver\"}[5m]))",
"legend": "{{ instance }} {{ code }}"
}
],
"name": "API Server - HTTP Requests by code",
"links": [],
"description": "按照返回码分类统计apiserver请求数",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 8,
"i": "1cfa42b1-9dcf-471c-90ff-8ffe656d4b11",
"isResizable": true
},
"id": "1cfa42b1-9dcf-471c-90ff-8ffe656d4b11"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by (instance,verb) (rate(apiserver_request_total{job=\"apiserver\"}[5m]))",
"legend": "{{ instance }} {{ verb }}"
}
],
"name": "API Server - HTTP Requests by verb",
"links": [],
"description": "按照请求动作分类统计apiserver的请求数",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 8,
"i": "94def0cb-0b86-42f7-a4b2-dde714bbb918",
"isResizable": true
},
"id": "94def0cb-0b86-42f7-a4b2-dde714bbb918"
},
{
"targets": [
{
"refId": "A",
"expr": "apiserver_current_inflight_requests{job=\"apiserver\"}",
"legend": "{{ instance }} {{ request_kind }}"
}
],
"name": "API Server - Current Inflight Requests by kind",
"links": [],
"description": "当前并发请求apiserver的数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 16,
"i": "ce5a15ad-11c6-44a2-a071-be57009162e1",
"isResizable": true
},
"id": "fb6266a3-3da0-4310-bfe8-c64a53db5db3"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) by (instance,verb,le))*1000",
"legend": "{{ instance }} {{ verb }}"
}
],
"name": "API Server - HTTP Requets Latency by verb",
"links": [],
"description": "apiserver的响应延迟，按请求动作分类统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 16,
"i": "045dca2d-d69b-47a7-b25e-656adb357e11",
"isResizable": true
},
"id": "045dca2d-d69b-47a7-b25e-656adb357e11"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket {job=\"apiserver\",verb!=\"WATCH\"}[5m])) by (instance,le))*1000",
"legend": "{{ instance }}"
}
],
"name": "API Server - HTTP Requets Latency by instance",
"links": [],
"description": "apiserver的响应延迟(非watch请求）",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 24,
"i": "1e775704-9ee4-45ce-9d24-b49af89fb5c7",
"isResizable": true
},
"id": "1e775704-9ee4-45ce-9d24-b49af89fb5c7"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance,verb) (rate(apiserver_request_total{code=~\"5..\",job=\"apiserver\"}[5m]))\n / sum by(instance,verb) (rate(apiserver_request_total{job=\"apiserver\"}[5m]))",
"legend": "{{ instance }} {{ verb }}"
}
],
"name": "API Server - Errors by verb",
"links": [],
"description": "apiserver的5xx错误率，按请求动作分类统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 24,
"i": "1ca62e0b-72df-47d1-93ba-048ed49e9cb5",
"isResizable": true
},
"id": "1ca62e0b-72df-47d1-93ba-048ed49e9cb5"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance) (rate(apiserver_request_total{code=~\"5..\", job=\"apiserver\"}[5m]))\n / sum by(instance) (rate(apiserver_request_total{job=\"apiserver\"}[5m]))",
"legend": "{{ instance }}"
}
],
"name": "API Server - Errors by Instance",
"links": [],
"description": "apiserver的5xx 错误率(5xx请求数/总请求数)",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 32,
"i": "92a209a1-7d30-4627-9ae1-55ded5095ed7",
"isResizable": true
},
"id": "92a209a1-7d30-4627-9ae1-55ded5095ed7"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(workqueue_depth{job=\"apiserver\"}[5m])) by (instance,name)",
"legend": "{{ instance }} {{ name }}"
}
],
"name": "API Server - Work Queue by instance",
"links": [],
"description": "apiserver工作队列深度，越接近0越好",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 32,
"i": "83f22cf4-9c65-4ad3-900b-fa6fc914dd88",
"isResizable": true
},
"id": "83f22cf4-9c65-4ad3-900b-fa6fc914dd88"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (instance)",
"legend": "{{ instance }}"
}
],
"name": "API Server - HTTP Requests by instance",
"links": [],
"description": "5分钟内apiserver的请求数统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "normal"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 40,
"i": "3e9f9df7-d9fb-4791-b3b2-2c52678f060f",
"isResizable": true
},
"id": "3e9f9df7-d9fb-4791-b3b2-2c52678f060f"
},
{
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_seconds_total{job=\"apiserver\"}[5m])",
"legend": "{{ instance }}"
}
],
"name": "API Server - CPU Usage by instance",
"links": [],
"description": "apiserver的cpu使用率",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "percent",
"decimals": 2
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 40,
"i": "3d5c1ae5-e640-4986-9202-78258169bffb",
"isResizable": true
},
"id": "3d5c1ae5-e640-4986-9202-78258169bffb"
},
{
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{job=\"apiserver\"}",
"legend": "{{ instance }}"
}
],
"name": "API Server - Memory Usage by instance",
"links": [],
"description": "apiserver的内存使用量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 48,
"i": "1550a2d5-c808-4174-865a-a41b2c16b486",
"isResizable": true
},
"id": "1550a2d5-c808-4174-865a-a41b2c16b486"
}
]
}
}
二、Controller-manager监控大盘
{
"name": "Kubernetes / Controller Manager",
"tags": "Categraf",
"configs": {
"version": "2.0.0",
"links": [],
"var": [],
"panels": [
{
"targets": [
{
"refId": "A",
"expr": "up{job=\"controller-manager\"}",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Health Status",
"links": [],
"description": "controller manager健康状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value"
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "UP"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#f60707",
"text": "DOWN"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 6,
"w": 12,
"x": 0,
"y": 0,
"i": "5d6560c5-6137-4632-bb88-ff8c9cf42e9d",
"isResizable": true
},
"id": "5d6560c5-6137-4632-bb88-ff8c9cf42e9d"
},
{
"targets": [
{
"refId": "A",
"expr": "leader_election_master_status{job=\"controller-manager\"}",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Member Status",
"links": [],
"description": "controller manager 主从状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value",
"result": {
"text": "val"
}
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "MASTER"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#9470ff",
"text": "BACKUP"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 6,
"w": 12,
"x": 12,
"y": 0,
"i": "a5d1ef0c-83e3-4194-b242-d5c51ba4bdd2",
"isResizable": true
},
"id": "62e3b249-fefe-4f32-8baf-394eac053f2a"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"controller-manager\"}[5m])) by (instance,code)",
"legend": "{{ instance }} {{ code }}"
}
],
"name": "Controller Manager - Requests by code",
"links": [],
"description": "controller manager 请求量 按返回码统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 6,
"i": "94713dc3-acb7-43b5-ae2f-399b2da61763",
"isResizable": true
},
"id": "94713dc3-acb7-43b5-ae2f-399b2da61763"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"controller-manager\"}[5m])) by (instance,method)",
"legend": "{{ instance }} {{ method }}"
}
],
"name": "Controller Manager - Requests by verb",
"links": [],
"description": "controller manager请求量，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 6,
"i": "a6928b49-cf0a-443e-a8fd-b999685df0be",
"isResizable": true
},
"id": "a6928b49-cf0a-443e-a8fd-b999685df0be"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"controller-manager\"}[5m])) by (instance)",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager -Requests by instance",
"links": [],
"description": "controller manager请求(apiserver)量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 14,
"i": "69690063-d044-4547-9f5f-126e5f8bf55a",
"isResizable": true
},
"id": "69690063-d044-4547-9f5f-126e5f8bf55a"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(rest_client_request_duration_seconds_bucket{job=\"controller-manager\"}[5m])) by (instance,verb,le))*1000",
"legend": "{{ instance }} {{ verb }}"
}
],
"name": "Controller Manager - Requests Latancy by verb",
"links": [],
"description": "controller manager 90分位请求apiserver延迟，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 14,
"i": "d5a67103-9930-46e0-97e1-296e0d71e30e",
"isResizable": true
},
"id": "053d10f3-1113-40e0-85aa-dfbabb706995"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(rest_client_request_duration_seconds_bucket{job=\"controller-manager\"}[5m])) by (instance,url,verb,le))*1000",
"legend": "{{ instance }} {{ verb }} {{ url }}"
}
],
"name": "Controller Manager - Requests Latancy by url",
"links": [],
"description": "controller manager90分位请求延迟，按url统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 22,
"i": "2924bb3f-20c3-4f56-96ff-76d473743d8b",
"isResizable": true
},
"id": "c86ed101-a91c-4478-b67a-7182a5e856d1"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(rest_client_request_duration_seconds_bucket{job=\"controller-manager\"}[5m])) by (instance,le)) * 1000",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Requests Latancy by instance",
"links": [],
"description": "controller manager90分位请求延迟，按实例统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 22,
"i": "75671720-bca1-449f-9c68-bf562f105b66",
"isResizable": true
},
"id": "80bd434e-21dc-4864-97c6-bfd1e2e27bbe"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance,method) (rate(rest_client_requests_total{code=~\"5..\",job=\"controller-manager\"}[5m]))\n / sum by(instance,method) (rate(rest_client_requests_total{job=\"controller-manager\"}[5m]))",
"legend": "{{ instance }} {{ method }}"
}
],
"name": "Controller Manager - Errors by verb",
"links": [],
"description": "controller manager请求5xx，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 30,
"i": "cfc389ad-5648-4107-a5bd-1680f6ede2ed",
"isResizable": true
},
"id": "32a09298-d0f8-4d54-808e-d223d0a428ff"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance) (rate(rest_client_requests_total{code=~\"5..\",job=\"controller-manager\"}[5m]))\n / sum by(instance) (rate(rest_client_requests_total{job=\"controller-manager\"}[5m]))",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Errors by instance",
"links": [],
"description": "controller manager请求5xx，按实例统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 30,
"i": "152b5817-ad87-44d0-a71f-5fbd0fc10ca3",
"isResizable": true
},
"id": "b6931f1f-6c43-478e-bcc7-26d1b121bceb"
},
{
"targets": [
{
"refId": "A",
"expr": "avg(rate(workqueue_adds_total{job=\"controller-manager\"}[5m])) by (instance,name)",
"legend": "{{ instance }} {{ name }}"
}
],
"name": "Controller Manager - Average Enqueue Rate by instance",
"links": [],
"description": "controller manager workqueue 添加任务数的速率",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 38,
"i": "0c8ac9ee-2a3e-4e7e-b338-748d79f6cbb6",
"isResizable": true
},
"id": "0c8ac9ee-2a3e-4e7e-b338-748d79f6cbb6"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(workqueue_depth{job=\"controller-manager\"}[5m])) by(instance,name)",
"legend": "{{ instance }} {{ name }} workqueue depth"
}
],
"name": "Controller Manager - WorkQueue Depth by name",
"links": [],
"description": "各个controller 队列深度",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 38,
"i": "b15af6b1-107d-4246-9eec-06fd370d4d35",
"isResizable": true
},
"id": "b15af6b1-107d-4246-9eec-06fd370d4d35"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"controller-manager\"}[5m])) by (instance,name,le))*1000",
"legend": "{{ instance }} {{ name }}"
}
],
"name": "Controller Manager - WorkQueue Queue Time by name",
"links": [],
"description": "任务在队列中的90分位等待耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 46,
"i": "1f2a42da-cdf8-4ce9-830f-92dabeb387be",
"isResizable": true
},
"id": "1b4705d3-cb3c-49c9-b60d-69edd28b662c"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(workqueue_work_duration_seconds_bucket{job=\"controller-manager\"}[5m])) by (instance,name,le))*1000",
"legend": "{{ instance }} {{ name }}"
}
],
"name": "Controller Manager - WorkQueue Work Time by name",
"links": [],
"description": "controller manager 90分位 任务出队到完成的耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 46,
"i": "30b7d514-fc44-4e24-9379-da697ceba79a",
"isResizable": true
},
"id": "1779a666-696e-4f07-b93f-f4b5bdd9d102"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(workqueue_retries_total{job=\"controller-manager\"}[5m])) by (instance,name)",
"legend": "{{ instance }} {{ name }}"
}
],
"name": "Controller Manager - WorkQueue Retries Total",
"links": [],
"description": "controller manager 任务进入队列的重试次数",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 54,
"i": "4841fbd4-8393-412f-ba40-7e63d79827e8",
"isResizable": true
},
"id": "084ccfb0-b7a1-4865-825d-f913bbc8456f"
},
{
"targets": [
{
"refId": "A",
"expr": "workqueue_longest_running_processor_seconds{job=\"controller-manager\"}*1000",
"legend": "{{ instance }} {{ name }}"
}
],
"name": "Controller Manager - WorkQueue longest running time",
"links": [],
"description": "队列中最耗时任务的处理时间（500ms采样周期，未完成的任务-任务开始处理即出队的时间)",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 54,
"i": "5c576ba5-0e85-4401-9e44-346b987eb8ba",
"isResizable": true
},
"id": "a1644e81-5aee-409b-be7a-6445f9478373"
},
{
"targets": [
{
"refId": "A",
"expr": "pv_collector_bound_pv_count{job=\"controller-manager\"}",
"legend": "{{ instance }} {{ storage_class }} bound pv "
},
{
"expr": "pv_collector_total_pv_count{job=\"controller-manager\"}",
"refId": "B",
"legend": "{{ instance }} total pv count"
}
],
"name": "Controller Manager - Bound PV Count",
"links": [],
"description": "controller manager 绑定的pv数目",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 62,
"i": "c28c023d-08e3-4614-8d20-0f0f3f5c7044",
"isResizable": true
},
"id": "d17fab46-1a62-47a5-9666-ee083f2ec9e5"
},
{
"targets": [
{
"refId": "A",
"expr": "node_collector_zone_size{job=\"controller-manager\"}",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Zone Size",
"links": [],
"description": "controller manager zone中的node数据",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 62,
"i": "8ccebf05-e22a-475e-9c4b-308887757855",
"isResizable": true
},
"id": "ce8b2909-215d-4a2a-a2ea-5faff34cf4ef"
},
{
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_seconds_total{job=\"controller-manager\"}[5m])",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - CPU Usage by instance",
"links": [],
"description": "controller manager cpu使用率",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "percentUnit",
"decimals": 2
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 70,
"i": "9c187c1c-f5cd-4aab-af81-09169948ab82",
"isResizable": true
},
"id": "9c187c1c-f5cd-4aab-af81-09169948ab82"
},
{
"targets": [
{
"refId": "A",
"expr": "pv_collector_bound_pvc_count{job=\"controller-manager\"}",
"legend": "{{ instance }} {{ namespace }} bound pvc"
},
{
"expr": "pv_collector_unbound_pvc_count{job=\"controller-manager\"}",
"refId": "B",
"legend": "{{ instance }} {{ namespace }} unbound pvc"
}
],
"name": "Controller Manager - PVC Count",
"links": [],
"description": "controller manager 绑定/非绑定的pvc数目",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 70,
"i": "6ad946ea-8cee-42f5-9863-3a76d53340a4",
"isResizable": true
},
"id": "92b035eb-2bd3-417d-b82a-f5734b8a4aec"
},
{
"targets": [
{
"refId": "A",
"expr": "process_open_fds{job=\"controller-manager\"}",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Open fds by instance",
"links": [],
"description": "controller manager打开的fd数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 78,
"i": "e438aed2-6d4a-4254-a8ec-26752385dc74",
"isResizable": true
},
"id": "94bb09a7-dbf7-41d8-b6a4-16b262365474"
},
{
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{job=\"controller-manager\"}",
"legend": "{{ instance }}"
}
],
"name": "Controller Manager - Memory Usage by instance",
"links": [],
"description": "controller manager 内存使用量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 78,
"i": "8cff2618-b2d4-4fb4-bfc2-d1d4c4f1b35c",
"isResizable": true
},
"id": "8cff2618-b2d4-4fb4-bfc2-d1d4c4f1b35c"
}
]
}
}
三、Scheduler监控大盘
{
"name": "Kubernetes / Scheduler",
"tags": "Categraf",
"configs": {
"version": "2.0.0",
"links": [],
"var": [],
"panels": [
{
"targets": [
{
"refId": "A",
"expr": "up{job=\"scheduler\"}",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Health Status",
"links": [],
"description": "scheduler健康状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value"
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "UP"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#f80202",
"text": "DOWN"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 6,
"w": 12,
"x": 0,
"y": 0,
"i": "5d6560c5-6137-4632-bb88-ff8c9cf42e9d",
"isResizable": true
},
"id": "5d6560c5-6137-4632-bb88-ff8c9cf42e9d"
},
{
"targets": [
{
"refId": "A",
"expr": "leader_election_master_status{job=\"scheduler\"}",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Member Status",
"links": [],
"description": "scheduler 主从状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value",
"result": {
"text": "val2"
}
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "MASTER"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#9470ff",
"text": "BACKUP"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 6,
"w": 12,
"x": 12,
"y": 0,
"i": "a5d1ef0c-83e3-4194-b242-d5c51ba4bdd2",
"isResizable": true
},
"id": "62e3b249-fefe-4f32-8baf-394eac053f2a"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"scheduler\"}[5m])) by (instance,code)",
"legend": "{{ instance }} {{ code }}"
}
],
"name": "Scheduler - Requests by code",
"links": [],
"description": "scheduler 请求量 按返回码统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 6,
"i": "94713dc3-acb7-43b5-ae2f-399b2da61763",
"isResizable": true
},
"id": "94713dc3-acb7-43b5-ae2f-399b2da61763"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"scheduler\"}[5m])) by (instance,method)",
"legend": "{{ instance }} {{ method }}"
}
],
"name": "Scheduler - Requests by verb",
"links": [],
"description": "scheduler请求量，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 6,
"i": "a6928b49-cf0a-443e-a8fd-b999685df0be",
"isResizable": true
},
"id": "a6928b49-cf0a-443e-a8fd-b999685df0be"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"scheduler\"}[5m])) by (instance)",
"legend": "{{ instance }}"
}
],
"name": "Scheduler -Requests by instance",
"links": [],
"description": "scheduler请求(apiserver)量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 14,
"i": "69690063-d044-4547-9f5f-126e5f8bf55a",
"isResizable": true
},
"id": "69690063-d044-4547-9f5f-126e5f8bf55a"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(rest_client_request_duration_seconds_bucket{job=\"scheduler\"}[5m])) by (instance,verb,le))*1000",
"legend": "{{ instance }} {{ verb }}"
}
],
"name": "Scheduler - Requests Latancy by verb",
"links": [],
"description": "scheduler 90分位请求延迟，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 14,
"i": "d5a67103-9930-46e0-97e1-296e0d71e30e",
"isResizable": true
},
"id": "053d10f3-1113-40e0-85aa-dfbabb706995"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(rest_client_request_duration_seconds_bucket{job=\"scheduler\"}[5m])) by (instance,url,verb,le))*1000",
"legend": "{{ instance }} {{ verb }} {{ url }}"
}
],
"name": "Scheduler - Requests Latancy by url",
"links": [],
"description": "scheduler 90分位请求延迟，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 22,
"i": "2924bb3f-20c3-4f56-96ff-76d473743d8b",
"isResizable": true
},
"id": "c86ed101-a91c-4478-b67a-7182a5e856d1"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(rest_client_request_duration_seconds_bucket{job=\"scheduler\"}[5m])) by (instance,le))*1000",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Requests Latancy by instance",
"links": [],
"description": "scheduler 90分位请求延迟，按实例统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 22,
"i": "75671720-bca1-449f-9c68-bf562f105b66",
"isResizable": true
},
"id": "80bd434e-21dc-4864-97c6-bfd1e2e27bbe"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance,method) (rate(rest_client_requests_total{code=~\"5..\",job=\"scheduler\"}[5m]))\n / sum by(instance,method) (rate(rest_client_requests_total{job=\"scheduler\"}[5m]))",
"legend": "{{ instance }} {{ method }}"
}
],
"name": "Scheduler - Errors by verb",
"links": [],
"description": "scheduler请求5xx，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 30,
"i": "cfc389ad-5648-4107-a5bd-1680f6ede2ed",
"isResizable": true
},
"id": "32a09298-d0f8-4d54-808e-d223d0a428ff"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance) (rate(rest_client_requests_total{code=~\"5..\",job=\"scheduler\"}[5m]))\n / sum by(instance) (rate(rest_client_requests_total{job=\"scheduler\"}[5m]))",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Errors by instance",
"links": [],
"description": "scheduler请求5xx，按实例统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 30,
"i": "152b5817-ad87-44d0-a71f-5fbd0fc10ca3",
"isResizable": true
},
"id": "b6931f1f-6c43-478e-bcc7-26d1b121bceb"
},
{
"targets": [
{
"refId": "A",
"expr": "avg(rate(scheduler_queue_incoming_pods_total{job=\"scheduler\"}[5m])) by (instance)",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Average Enqueue Rate by instance",
"links": [],
"description": "pod进入调度队列的平均速率",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 38,
"i": "0c8ac9ee-2a3e-4e7e-b338-748d79f6cbb6",
"isResizable": true
},
"id": "0c8ac9ee-2a3e-4e7e-b338-748d79f6cbb6"
},
{
"targets": [
{
"refId": "A",
"expr": "avg(rate(scheduler_preemption_attempts_total{job=\"scheduler\"}[5m])) by (instance)",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Preemption Attempts Total by instance",
"links": [],
"description": "调度器驱逐容器的次数",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 38,
"i": "730be715-cfdd-4f2a-b878-1b2fb9a9be45",
"isResizable": true
},
"id": "ed72c89c-9732-4fb0-9187-0cabe9a4a81c"
},
{
"targets": [
{
"refId": "A",
"expr": " scheduler_scheduler_cache_size{job=\"scheduler\"}",
"legend": "{{ instance }} {{ type }}"
}
],
"name": "Scheduler - Cache Size",
"links": [],
"description": "调度器cache中pod node和已绑定pod的数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 46,
"i": "668362fe-211a-4286-861c-eeb90907b2fb",
"isResizable": true
},
"id": "dd189a18-704b-44c0-8a8b-186ccc591c81"
},
{
"targets": [
{
"refId": "A",
"expr": "scheduler_pending_pods{job=\"scheduler\"}",
"legend": "{{ instance }} {{ queue }}"
}
],
"name": "Scheduler - Pending Pod",
"links": [],
"description": "调度队列中pending pod的数目",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 46,
"i": "f20223e4-199a-4b08-b5f0-470ac1974afc",
"isResizable": true
},
"id": "0e231e69-4651-4728-abcb-46fafd1a7d61"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(scheduler_plugin_execution_duration_seconds_bucket{job=\"scheduler\"}[5m])) by (instance,extension_point,plugin,status,le))*1000",
"legend": "{{ instance }} {{ plugin }} {{ extension_point }} {{ status }}"
}
],
"name": "Scheduler - Plugin Execution Duration by plugin",
"links": [],
"description": "调度插件在每个扩展点的执行时间,90分位",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 54,
"i": "ec0db15d-fa63-4448-a961-0b8a7b977dd0",
"isResizable": true
},
"id": "51cb0215-0827-4543-bb58-64eaec6cdc77"
},
{
"targets": [
{
"expr": "histogram_quantile(0.9, sum (rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"scheduler\"}[5m])) by (instance,le))*1000",
"refId": "B",
"legend": "{{ instance }} "
}
],
"name": "Scheduler - Scheduling Algorithm Duration by instance",
"links": [],
"description": "调度算法90分位耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 54,
"i": "c5c557a6-0f48-4154-91f7-30a300f00813",
"isResizable": true
},
"id": "bf41347e-c307-4ae1-84d8-6202b563837f"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(scheduler_scheduling_attempt_duration_seconds_bucket{job=\"scheduler\"}[5m])) by (instance,le,profile,result))*1000",
"legend": "{{ instance }} {{ profile }} {{ result }}"
}
],
"name": "Scheduler - Scheduling Duration by instance",
"links": [],
"description": "调度算法+绑定 90分位耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 62,
"i": "68585816-6a08-422d-a7d3-c414ae427380",
"isResizable": true
},
"id": "4f2de85b-7fa7-4e06-bcbf-7362fa71f010"
},
{
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_seconds_total{job=\"scheduler\"}[5m])",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - CPU Usage by instance",
"links": [],
"description": "scheduler cpu使用率",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "percentUnit",
"decimals": 2
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 62,
"i": "9c187c1c-f5cd-4aab-af81-09169948ab82",
"isResizable": true
},
"id": "9c187c1c-f5cd-4aab-af81-09169948ab82"
},
{
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{job=\"scheduler\"}",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Memory Usage by instance",
"links": [],
"description": "scheduler 内存使用量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 70,
"i": "8cff2618-b2d4-4fb4-bfc2-d1d4c4f1b35c",
"isResizable": true
},
"id": "8cff2618-b2d4-4fb4-bfc2-d1d4c4f1b35c"
},
{
"targets": [
{
"refId": "A",
"expr": "process_open_fds{job=\"scheduler\"}",
"legend": "{{ instance }}"
}
],
"name": "Scheduler - Open fds by instance",
"links": [],
"description": "scheduler打开的fd数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 70,
"i": "e438aed2-6d4a-4254-a8ec-26752385dc74",
"isResizable": true
},
"id": "94bb09a7-dbf7-41d8-b6a4-16b262365474"
}
]
}
}
四、Etcd监控大盘
{
"name": "Kubernetes / Etcd",
"tags": "Categraf",
"configs": {
"version": "2.0.0",
"links": [],
"var": [
{
"name": "job",
"definition": "label_values(etcd_server_is_leader, job)"
}
],
"panels": [
{
"targets": [
{
"refId": "A",
"expr": "up{job=\"$job\"}",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Health Status",
"links": [],
"description": "etcd健康状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value"
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "UP"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#000000",
"text": "DOWN"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 6,
"w": 12,
"x": 0,
"y": 0,
"i": "5d6560c5-6137-4632-bb88-ff8c9cf42e9d",
"isResizable": true
},
"id": "5d6560c5-6137-4632-bb88-ff8c9cf42e9d"
},
{
"targets": [
{
"refId": "A",
"expr": "etcd_server_is_leader{job=\"$job\"} * on (instance) group_left(server_id) etcd_server_id{job=\"$job\"}",
"legend": ""
}
],
"name": "Etcd - Member Status",
"links": [],
"description": "etcd snapshot 状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "labelsOfSeriesToRows",
"columns": [
"instance",
"server_id",
"value"
]
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value",
"result": {
"text": ""
}
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "LEADER"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#e90e17",
"text": "LERANER"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 6,
"w": 12,
"x": 12,
"y": 0,
"i": "2d6c513d-5553-4635-a3f7-ec5955deb009",
"isResizable": true
},
"id": "7480f8ad-c5a5-48e8-aec1-c6347beb3f38"
},
{
"targets": [
{
"refId": "A",
"expr": "etcd_server_health_success{job=\"$job\"}",
"legend": "{{ instance }} - check success"
},
{
"expr": "etcd_server_health_failures{job=\"$job\"}",
"refId": "B",
"legend": "{{ instance }} - check failure"
}
],
"name": "Etcd - Health Check Stats",
"links": [],
"description": "etcd 健康检查状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 6,
"i": "e8794091-aff6-4dbf-9876-685a6f277794",
"isResizable": true
},
"id": "db5575c1-0859-4b42-b327-5dd57f1e1225"
},
{
"targets": [
{
"refId": "A",
"expr": "etcd_disk_defrag_inflight{job=\"$job\"}",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Defrag Status",
"links": [],
"description": "etcd defrag（压缩） 状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value",
"result": {
"text": ""
}
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "DEFRAG"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#9470ff",
"text": "-"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 6,
"i": "ce4a873d-65f0-4071-868c-e67d6b723ab6",
"isResizable": true
},
"id": "69d1bab4-1b37-4ece-9630-d8e3e105ce4e"
},
{
"targets": [
{
"refId": "A",
"expr": "etcd_server_snapshot_apply_in_progress_total{job=\"$job\"}",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Snapshot Status",
"links": [],
"description": "etcd snapshot（快照） 状态",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value",
"result": {
"text": ""
}
},
{
"type": "special",
"result": {
"color": "#3fc453",
"text": "APPLYING"
},
"match": {
"special": 1
}
},
{
"type": "special",
"result": {
"color": "#9470ff",
"text": "-"
},
"match": {
"special": 0
}
}
],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 14,
"i": "a5d1ef0c-83e3-4194-b242-d5c51ba4bdd2",
"isResizable": true
},
"id": "62e3b249-fefe-4f32-8baf-394eac053f2a"
},
{
"targets": [
{
"refId": "A",
"expr": "etcd_server_leader_changes_seen_total{job=\"$job\"}",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Leader Change Stats",
"links": [],
"description": "etcd leader切换统计",
"custom": {
"showHeader": true,
"colorMode": "value",
"calc": "lastNotNull",
"displayMode": "seriesToRows"
},
"options": {
"valueMappings": [],
"standardOptions": {
"util": "none"
}
},
"overrides": [
{
"properties": {
"valueMappings": []
}
}
],
"version": "2.0.0",
"type": "table",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 14,
"i": "25b5262d-a318-4489-9dae-152d31906704",
"isResizable": true
},
"id": "be940718-c4ac-4f6f-a966-dd1961a9e019"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(grpc_server_handled_total{job=\"$job\"}[5m])) by (instance,grpc_code)",
"legend": "{{ instance }} {{ grpc_code }}"
}
],
"name": "Etcd - Requests by code",
"links": [],
"description": "etcd 处理的请求数，按响应码统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 22,
"i": "94713dc3-acb7-43b5-ae2f-399b2da61763",
"isResizable": true
},
"id": "94713dc3-acb7-43b5-ae2f-399b2da61763"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(grpc_server_handled_total{job=\"$job\"}[5m])) by (instance)",
"legend": "{{ instance }}"
}
],
"name": "Etcd -Requests by instance",
"links": [],
"description": "etcd处理的请求数， 按实例统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 22,
"i": "69690063-d044-4547-9f5f-126e5f8bf55a",
"isResizable": true
},
"id": "69690063-d044-4547-9f5f-126e5f8bf55a"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(grpc_server_handled_total{job=\"$job\",grpc_method=~\"Compact|Defrag.*|Member.*|Put|Watch|Lease.*|Range|Snapshot|Txn|Hash.*|Status|Auth.*\"}[5m])) by (instance,grpc_method)",
"legend": "{{ instance }} {{ grpc_method }}"
}
],
"name": "Etcd - Requests by verb",
"links": [],
"description": "etcd处理的请求数，按请求类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 30,
"i": "a6928b49-cf0a-443e-a8fd-b999685df0be",
"isResizable": true
},
"id": "a6928b49-cf0a-443e-a8fd-b999685df0be"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$job\"}[5m])) by (instance,le))*1000",
"legend": "{{ instance }}"
}
],
"name": "Etcd - WAL Fsync Latancy by instance",
"links": [],
"description": "etcd 90分位WAL fsync耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 30,
"i": "75671720-bca1-449f-9c68-bf562f105b66",
"isResizable": true
},
"id": "80bd434e-21dc-4864-97c6-bfd1e2e27bbe"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$job\"}[5m])) by (instance,le))*1000",
"legend": "{{ instance }} "
}
],
"name": "Etcd - Commit Latancy by instance",
"links": [],
"description": "etcd 90分位commit耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 38,
"i": "d5a67103-9930-46e0-97e1-296e0d71e30e",
"isResizable": true
},
"id": "053d10f3-1113-40e0-85aa-dfbabb706995"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(etcd_disk_backend_defrag_duration_seconds_bucket{job=\"$job\"}[5m])) by (instance,le))*1000",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Defrag Latancy by instance",
"links": [],
"description": "etcd 90分位defrag耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 38,
"i": "2924bb3f-20c3-4f56-96ff-76d473743d8b",
"isResizable": true
},
"id": "c86ed101-a91c-4478-b67a-7182a5e856d1"
},
{
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.9, sum(rate(etcd_disk_backend_snapshot_duration_seconds_bucket{job=\"$job\"}[5m])) by (instance,le))*1000",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Snapshot Latancy by instance",
"links": [],
"description": "etcd 90分位snapshot耗时",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 46,
"i": "e4ec9053-5700-4ede-abfa-35f115597665",
"isResizable": true
},
"id": "5c6c48fd-b12b-49d7-86df-b86d90134339"
},
{
"targets": [
{
"refId": "A",
"expr": "sum by(instance) (rate(etcd_server_proposals_failed_total{job=\"$job\"}[5m]))",
"legend": "{{ instance }} failure"
},
{
"expr": "etcd_server_proposals_pending{job=\"$job\"}",
"refId": "B",
"legend": "{{ instance }} pending"
}
],
"name": "Etcd - Proposal Failures/Pending by instance",
"links": [],
"description": "etcd协商失败/pending次数",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 46,
"i": "152b5817-ad87-44d0-a71f-5fbd0fc10ca3",
"isResizable": true
},
"id": "b6931f1f-6c43-478e-bcc7-26d1b121bceb"
},
{
"targets": [
{
"refId": "A",
"expr": "sum (rate(etcd_server_read_indexes_failed_total{job=\"$job\"}[5m])) by (instance)",
"legend": "{{ instance }} failure read"
},
{
"expr": "sum(rate(etcd_server_slow_read_indexes_total{job=\"$job\"}[5m])) by (instance)",
"refId": "B",
"legend": "{{ instance }} slow read"
}
],
"name": "Etcd - Server Read Indexes Failure / Slow Read",
"links": [],
"description": "etcd读取索引失败 读取索引慢（过期索引或者超时）次数",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 54,
"i": "784d72f7-fb4e-46e3-b632-e70df6dde0a7",
"isResizable": true
},
"id": "369af7f4-9851-4c8c-a186-04db4fa7f90f"
},
{
"targets": [
{
"refId": "A",
"expr": "etcd_server_quota_backend_bytes{job=\"$job\"}",
"legend": "{{ instance }} quota"
},
{
"expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$job\"}",
"refId": "B",
"legend": "{{ instance }} total size"
},
{
"expr": "etcd_mvcc_db_total_size_in_use_in_bytes",
"refId": "C",
"legend": "{{ instance }} in use"
}
],
"name": "Etcd - MVCC DB Size",
"links": [],
"description": "etcd 存储quota/total/in use大小",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 54,
"i": "730be715-cfdd-4f2a-b878-1b2fb9a9be45",
"isResizable": true
},
"id": "ed72c89c-9732-4fb0-9187-0cabe9a4a81c"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(etcd_mvcc_range_total{job=\"$job\"}[5m])) by (instance)",
"legend": "{{ instance }} range"
},
{
"expr": "sum(rate(etcd_mvcc_put_total{job=\"$job\"}[5m])) by (instance)",
"refId": "B",
"legend": "{{ instance }} put"
},
{
"expr": "sum(rate(etcd_mvcc_delete_total{job=\"$job\"}[5m])) by (instance)",
"refId": "C",
"legend": "{{ instance }} delete"
},
{
"expr": "sum(rate(etcd_mvcc_txn_total{job=\"$job\"}[5m])) by (instance)",
"refId": "D",
"legend": "{{ instance }} txn"
}
],
"name": "Etcd - Operations Stats by instance",
"links": [],
"description": "etcd操作类型统计",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "milliseconds"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 62,
"i": "cfc389ad-5648-4107-a5bd-1680f6ede2ed",
"isResizable": true
},
"id": "32a09298-d0f8-4d54-808e-d223d0a428ff"
},
{
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_seconds_total{job=\"$job\"}[5m])",
"legend": "{{ instance }}"
}
],
"name": "Etcd - CPU Usage by instance",
"links": [],
"description": "etcd cpu使用率",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "percentUnit",
"decimals": 2
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 62,
"i": "9c187c1c-f5cd-4aab-af81-09169948ab82",
"isResizable": true
},
"id": "9c187c1c-f5cd-4aab-af81-09169948ab82"
},
{
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{job=\"$job\"}",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Memory Usage by instance",
"links": [],
"description": "etcd 内存使用量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 0,
"y": 70,
"i": "8cff2618-b2d4-4fb4-bfc2-d1d4c4f1b35c",
"isResizable": true
},
"id": "8cff2618-b2d4-4fb4-bfc2-d1d4c4f1b35c"
},
{
"targets": [
{
"refId": "A",
"expr": "process_open_fds{job=\"$job\"}",
"legend": "{{ instance }}"
}
],
"name": "Etcd - Open fds by instance",
"links": [],
"description": "etcd打开的fd数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "list"
},
"standardOptions": {
"util": "none"
},
"thresholds": {
"steps": []
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.25,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 8,
"w": 12,
"x": 12,
"y": 70,
"i": "e438aed2-6d4a-4254-a8ec-26752385dc74",
"isResizable": true
},
"id": "94bb09a7-dbf7-41d8-b6a4-16b262365474"
}
]
}
}
五、Kubelet监控大盘
{
"name": "Kubelet metrics by ident",
"tags": "",
"ident": "",
"configs": {
"var": [
{
"name": "ident",
"type": "query",
"definition": "label_values(kubelet_running_pods, ident)",
"multi": true,
"allOption": true
}
],
"panels": [
{
"type": "stat",
"id": "d3caf396-b3a1-449b-acec-f550967889e6",
"layout": {
"h": 3,
"w": 4,
"x": 0,
"y": 0,
"i": "d3caf396-b3a1-449b-acec-f550967889e6",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "sum(up{job=\"kubelet\"})"
}
],
"name": "Kubelet UP",
"custom": {
"textMode": "value",
"colorMode": "background",
"calc": "lastNotNull",
"valueField": "Value",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
}
},
{
"targets": [
{
"refId": "A",
"expr": "sum(kubelet_running_pods{ident=~\"$ident\"})"
}
],
"name": "Running Pods",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 3,
"w": 4,
"x": 4,
"y": 0,
"i": "38c38b23-a7e3-4177-8c41-3ce955ea0434",
"isResizable": true
},
"id": "38c38b23-a7e3-4177-8c41-3ce955ea0434"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(kubelet_running_containers{ident=~\"$ident\", container_state=\"running\"})"
}
],
"name": "Running Containers",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 3,
"w": 4,
"x": 8,
"y": 0,
"i": "26bf2320-fcff-48f8-a6fc-aa9076bb9329",
"isResizable": true
},
"id": "525859b9-91d7-4180-b363-bf8ceec977d8"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(volume_manager_total_volumes{ident=~\"$ident\", state=\"desired_state_of_world\"})"
}
],
"name": "Desired Volumes",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 3,
"w": 4,
"x": 12,
"y": 0,
"i": "54ae4ab3-e932-418c-a637-f2f515cce1b9",
"isResizable": true
},
"id": "84af4617-2ae0-4b30-a82a-6e8586342224"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(volume_manager_total_volumes{ident=~\"$ident\", state=\"actual_state_of_world\"})"
}
],
"name": "Actual Volumes",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 3,
"w": 4,
"x": 16,
"y": 0,
"i": "d9de76d7-2203-40e7-a792-9888ec869e82",
"isResizable": true
},
"id": "d431f4bd-9115-41d2-a494-1d680bdd1e0f"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(increase(kubelet_runtime_operations_errors_total{ident=~\"$ident\"}[5m]))"
}
],
"name": "OP Errors in 5min",
"custom": {
"textMode": "value",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"valueMappings": [
{
"type": "range",
"match": {
"from": 1
},
"result": {
"color": "#d0021b"
}
},
{
"type": "range",
"match": {
"to": 1
},
"result": {
"color": "#417505"
}
}
],
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 3,
"w": 4,
"x": 20,
"y": 0,
"i": "bf2bbd15-347d-404c-9b8f-e524875befe2",
"isResizable": true
},
"id": "54de62bc-8af3-4c27-8b8e-1af567b363fc"
},
{
"type": "row",
"id": "730d4a9b-791f-4aaf-a042-668f66e73814",
"name": "Operations",
"collapsed": true,
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 3,
"i": "730d4a9b-791f-4aaf-a042-668f66e73814",
"isResizable": false
},
"panels": []
},
{
"targets": [
{
"refId": "A",
"expr": "increase(kubelet_runtime_operations_total{ident=~\"$ident\"}[5m])"
}
],
"name": "Operations in 5min",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 4,
"w": 12,
"x": 0,
"y": 4,
"i": "d26e6818-6704-492a-8cbf-58473dd85716",
"isResizable": true
},
"id": "d26e6818-6704-492a-8cbf-58473dd85716"
},
{
"targets": [
{
"refId": "A",
"expr": "increase(kubelet_runtime_operations_errors_total{ident=~\"$ident\"}[5m])"
}
],
"name": "Operation Errors in 5min",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 4,
"w": 12,
"x": 12,
"y": 4,
"i": "4e585d2f-c61c-4350-86ec-dca7ddc34ceb",
"isResizable": true
},
"id": "09a6ad5b-8c0e-4f17-b17f-3ebc514f7d20"
},
{
"targets": [
{
"refId": "A",
"expr": "increase(kubelet_runtime_operations_duration_seconds_sum{ident=~\"$ident\"}[1h])/increase(kubelet_runtime_operations_duration_seconds_count{ident=~\"$ident\"}[1h])"
}
],
"name": "Average Operation duration in 1 hour (Unit: Second)",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 4,
"w": 24,
"x": 0,
"y": 8,
"i": "b5e56f3e-fa20-4c19-8578-c0610fa0a7e7",
"isResizable": true
},
"id": "b5e56f3e-fa20-4c19-8578-c0610fa0a7e7"
},
{
"type": "row",
"id": "dd7e84c5-03ce-467c-871a-aa110fe051f4",
"name": "PLEG relist",
"collapsed": true,
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 12,
"i": "dd7e84c5-03ce-467c-871a-aa110fe051f4",
"isResizable": false
}
},
{
"targets": [
{
"refId": "A",
"expr": "rate(kubelet_pleg_relist_duration_seconds_count{ident=~\"$ident\"}[1h])"
}
],
"name": "relist rate",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 4,
"w": 12,
"x": 0,
"y": 13,
"i": "f3822da8-a9c9-4db1-ba12-465d3ece823e",
"isResizable": true
},
"id": "f3822da8-a9c9-4db1-ba12-465d3ece823e"
},
{
"targets": [
{
"refId": "A",
"expr": "increase(kubelet_pleg_relist_duration_seconds_sum{ident=~\"$ident\"}[1h])/increase(kubelet_pleg_relist_duration_seconds_count{ident=~\"$ident\"}[1h])"
}
],
"name": "relist duration (Unit: Second)",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 4,
"w": 12,
"x": 12,
"y": 13,
"i": "2b4ada76-6c30-42cd-9bd3-c939b4c0139c",
"isResizable": true
},
"id": "a6e4c914-bfca-4419-a264-f5b1cbab261a"
}
],
"version": "2.0.0"
}
}
六、Kube-proxy监控大盘
{
"name": "Kubernetes / Proxy",
"tags": "",
"ident": "",
"configs": {
"version": "2.0.0",
"links": [],
"var": [
{
"name": "ident",
"type": "query",
"definition": "label_values(kubeproxy_sync_proxy_rules_iptables_restore_failures_total, ident)",
"reg": "",
"multi": true,
"allOption": true,
"allValue": ".*"
}
],
"panels": [
{
"version": "2.0.0",
"id": "c0305f2f-68a1-4e60-9713-4d83f9cfd98c",
"type": "stat",
"name": "Up",
"links": [],
"layout": {
"h": 5,
"w": 4,
"x": 0,
"y": 0,
"i": "c0305f2f-68a1-4e60-9713-4d83f9cfd98c",
"isResizable": true
},
"targets": [
{
"refId": "A",
"expr": "sum(up{job=\"kube-proxy\"})",
"legend": ""
}
],
"options": {},
"custom": {
"version": "2.0.0",
"textMode": "value",
"colorMode": "value"
}
},
{
"type": "timeseries",
"id": "54bd34da-ea68-441d-ac6f-cd74b2130e14",
"layout": {
"h": 5,
"w": 10,
"x": 4,
"y": 0,
"i": "54bd34da-ea68-441d-ac6f-cd74b2130e14",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"kube-proxy\", ident=~\"$ident\"}[5m]))",
"legend": "rate"
}
],
"name": "Rules Sync Rate",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "20979f64-9057-4645-9328-3f4d1eb5e90f",
"layout": {
"h": 5,
"w": 10,
"x": 14,
"y": 0,
"i": "20979f64-9057-4645-9328-3f4d1eb5e90f",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"kube-proxy\", ident=~\"$ident\"}[5m]))",
"legend": "{{ident}}"
}
],
"name": "Rule Sync Latency 99th Quantile",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "seconds"
},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "e15d3349-b056-4df2-a6a2-52ddc7bed913",
"layout": {
"h": 5,
"w": 12,
"x": 0,
"y": 5,
"i": "e15d3349-b056-4df2-a6a2-52ddc7bed913",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"kube-proxy\", ident=~\"$ident\"}[5m]))",
"legend": "rate"
}
],
"name": "Network Programming Rate",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "4d63cd62-3e4f-489a-87cc-fea18ee9e4fe",
"layout": {
"h": 5,
"w": 12,
"x": 12,
"y": 5,
"i": "4d63cd62-3e4f-489a-87cc-fea18ee9e4fe",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\", ident=~\"$ident\"}[5m])) by (ident, le))",
"legend": "{{ident}}"
}
],
"name": "Network Programming Latency 99th Quantile",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "8ddc5ec1-4ccb-421c-9976-1a3c0cd416b8",
"layout": {
"h": 5,
"w": 12,
"x": 0,
"y": 10,
"i": "8ddc5ec1-4ccb-421c-9976-1a3c0cd416b8",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", ident=~\"$ident\",code=~\"2..\"}[5m]))",
"legend": "2xx"
},
{
"refId": "B",
"expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", ident=~\"$ident\",code=~\"3..\"}[5m]))",
"legend": "3xx"
},
{
"refId": "C",
"expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", ident=~\"$ident\",code=~\"4..\"}[5m]))",
"legend": "4xx"
},
{
"refId": "D",
"expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", ident=~\"$ident\",code=~\"5..\"}[5m]))",
"legend": "5xx"
}
],
"name": "Kube API Request Rate",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "ed034ce6-e6ec-4b5b-ad81-660d087c5cf9",
"layout": {
"h": 5,
"w": 12,
"x": 12,
"y": 10,
"i": "ed034ce6-e6ec-4b5b-ad81-660d087c5cf9",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-proxy\",ident=~\"$ident\",verb=\"POST\"}[5m])) by (verb, url, le))",
"legend": "{{verb}} {{url}}"
}
],
"name": "Post Request Latency 99th Quantile",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "f40ae278-17d1-45e3-848a-06f14ad0423d",
"layout": {
"h": 5,
"w": 24,
"x": 0,
"y": 15,
"i": "f40ae278-17d1-45e3-848a-06f14ad0423d",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-proxy\", ident=~\"$ident\", verb=\"GET\"}[5m])) by (verb, url, le))",
"legend": "{{verb}} {{url}}"
}
],
"name": "Get Request Latency 99th Quantile",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "047b27f1-7ff9-43ff-bc94-396688422071",
"layout": {
"h": 5,
"w": 8,
"x": 0,
"y": 20,
"i": "047b27f1-7ff9-43ff-bc94-396688422071",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{job=\"kube-proxy\",ident=~\"$ident\"}",
"legend": "{{ident}}"
}
],
"name": "Memory",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC",
"decimals": 1
},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "5103d5ee-2245-45be-ad70-f5410dd02544",
"layout": {
"h": 5,
"w": 8,
"x": 8,
"y": 20,
"i": "5103d5ee-2245-45be-ad70-f5410dd02544",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_seconds_total{job=\"kube-proxy\",ident=~\"$ident\"}[5m])",
"legend": "{{ident}}"
}
],
"name": "CPU usage",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
},
{
"type": "timeseries",
"id": "1d15c866-c83b-408b-98e7-e26ada5e7cfc",
"layout": {
"h": 5,
"w": 8,
"x": 16,
"y": 20,
"i": "1d15c866-c83b-408b-98e7-e26ada5e7cfc",
"isResizable": true
},
"version": "2.0.0",
"datasourceCate": "prometheus",
"targets": [
{
"refId": "A",
"expr": "go_goroutines{job=\"kube-proxy\",ident=~\"$ident\"}",
"legend": "{{ident}}"
}
],
"name": "Goroutines",
"links": [],
"options": {
"tooltip": {
"mode": "all",
"sort": "desc"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {
"steps": [
{
"color": "#634CD9",
"value": null,
"type": "base"
}
]
}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "linear",
"spanNulls": false,
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off",
"scaleDistribution": {
"type": "linear"
}
}
}
]
}
}
七、kube-state-metrics(KSM)资源监控大盘
{
"name": "Container - 模板",
"tags": "",
"configs": {
"var": [
{
"name": "pod_name",
"definition": "label_values(container_cpu_usage_seconds_total, pod)",
"multi": false,
"reg": ""
}
],
"version": "2.0.0",
"panels": [
{
"id": "79d7e3b0-b64f-4591-b5dd-994ce16b68ca",
"type": "row",
"name": "整体概况",
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 0,
"i": "79d7e3b0-b64f-4591-b5dd-994ce16b68ca"
},
"collapsed": true
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_usage_seconds_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])*100) by(name)",
"legend": ""
}
],
"name": "容器CPU使用率",
"description": "容器1min CPU平均使用率，如果pod内有多个容器，会分别显示各个容器CPU使用率（pause容器默认不会显示）；如果容器配置了4核，使用率400%表示当前用满4核，容器配置2核，使用率200%表示当前用满2核。",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"min": 0,
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 1,
"i": "860c1484-1f83-497e-a061-a50fbb3ff1dc"
},
"id": "860c1484-1f83-497e-a061-a50fbb3ff1dc"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_usage_seconds_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])*100) by(name)/((sum(container_spec_cpu_quota{pod=\"$pod_name\"}/container_spec_cpu_period{pod=\"$pod_name\"}) by (name)))",
"legend": ""
}
],
"name": "容器CPU归一化后使用率",
"description": "容器1min CPU平均使用率，如果pod内有多个容器，会分别显示各个容器CPU使用率（pause容器默认不会显示）；如果容器配置了4核，使用率100%表示当前用满4核，容器配置2核，使用率100%表示当前用满2核，如果容器没有配置cpu limit，则不会显示该数值",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"min": 0,
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 1,
"i": "bb66b2cc-3658-4cae-817a-61ec3fbb93e4"
},
"id": "bb66b2cc-3658-4cae-817a-61ec3fbb93e4"
},
{
"targets": [
{
"refId": "A",
"expr": "(sum(container_memory_usage_bytes{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name)) "
}
],
"name": "容器内存使用",
"description": "container_memory_rss + container_memory_cache + container_memory_swap + kernel memory",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC",
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 1,
"i": "4d5f9cbb-3b78-4fe6-8a4e-59ca52a49666"
},
"id": "4d5f9cbb-3b78-4fe6-8a4e-59ca52a49666"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_fs_writes_bytes_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])) by(name)",
"legend": ""
}
],
"name": "文件系统写入速率",
"description": "",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC",
"min": 0,
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 1,
"i": "978e93a0-770e-42f5-a374-cafb5b4fc585"
},
"id": "978e93a0-770e-42f5-a374-cafb5b4fc585"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_fs_reads_bytes_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])) by(name)",
"legend": ""
}
],
"name": "文件系统读取速率",
"description": "",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC",
"min": 0,
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 3,
"i": "f385736d-fd05-4705-a27d-41e67fb6c843"
},
"id": "f385736d-fd05-4705-a27d-41e67fb6c843"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_transmit_bytes_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络发送速率",
"description": "容器网络发送的字节数（1 分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 3,
"i": "f9b140e8-0b44-4b32-9d54-9360b06faa48"
},
"id": "f9b140e8-0b44-4b32-9d54-9360b06faa48"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_receive_bytes_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络接收速率",
"description": "容器网络接收的字节数（1 分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 3,
"i": "8f98a264-7058-4fa0-8efc-3c87954a0370"
},
"id": "8f98a264-7058-4fa0-8efc-3c87954a0370"
},
{
"targets": [
{
"refId": "A",
"expr": "sum((time()-container_start_time_seconds{pod=\"$pod_name\", image!~\".*pause.*\"})) by (name)"
}
],
"name": "容器启动时长（小时）",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {
"util": "humantimeSeconds"
}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 3,
"i": "1868973e-c3d3-4fd4-83b0-36ec2e06dfe0"
},
"id": "1868973e-c3d3-4fd4-83b0-36ec2e06dfe0"
},
{
"id": "8a4942a2-81a3-4473-81be-79b3541e09a3",
"type": "row",
"name": "CPU",
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 10,
"i": "8a4942a2-81a3-4473-81be-79b3541e09a3"
},
"collapsed": true
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_usage_seconds_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])*100) by(name)"
}
],
"name": "容器CPU使用率",
"description": "容器1min CPU平均使用率，如果pod内有多个容器，会分别显示各个容器CPU使用率（pause容器默认不会显示）；如果容器配置了4核，使用率400%表示当前用满4核，容器配置2核，使用率200%表示当前用满2核。",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 11,
"i": "54650fe9-007f-4b16-a523-baf2e91ef823"
},
"id": "54650fe9-007f-4b16-a523-baf2e91ef823"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_usage_seconds_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])*100) by(name)/((sum(container_spec_cpu_quota{pod=\"$pod_name\"}/container_spec_cpu_period{pod=\"$pod_name\"}) by (name)))"
}
],
"name": "容器CPU归一化后使用率",
"description": "容器1min CPU平均使用率，如果pod内有多个容器，会分别显示各个容器CPU使用率（pause容器默认不会显示）；如果容器配置了4核，使用率100%表示当前用满4核，容器配置2核，使用率100%表示当前用满2核，如果容器没有配置cpu limit，则不会显示该数值",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 11,
"i": "715c1e5a-c504-4f2c-a790-dad1c73aae29"
},
"id": "715c1e5a-c504-4f2c-a790-dad1c73aae29"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_system_seconds_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])*100) by(name)"
}
],
"name": "容器内核态CPU使用率",
"description": "容器1min 内核态CPU平均使用率，如果pod内有多个容器，会分别显示各个容器内核态CPU使用率（pause容器默认不会显示）；如果容器配置了4核，使用率400%表示当前内核态用满4核，容器配置2核，使用率200%表示当前内核态用满2核。",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 11,
"i": "d23a509d-3bb0-4680-b579-b89a411830a4"
},
"id": "d23a509d-3bb0-4680-b579-b89a411830a4"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_user_seconds_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])*100) by(name)"
}
],
"name": "容器用户态CPU使用率",
"description": "容器1min 内核态CPU平均使用率，如果pod内有多个容器，会分别显示各个容器用户态CPU使用率（pause容器默认不会显示）；如果容器配置了4核，使用率400%表示当前用户态用满4核，容器配置2核，使用率200%表示当前用户态用满2核。",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 11,
"i": "2921e6d0-1b9c-449d-9dd8-fb29dce8ca7d"
},
"id": "2921e6d0-1b9c-449d-9dd8-fb29dce8ca7d"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_cpu_cfs_throttled_periods_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m]))by(name) *100"
}
],
"name": "容器发生CPU throttle的比率",
"description": "该值大于0，说明CPU在运行期间发生throttle情况，即容器设置的CPU规格，不满足容器当前对CPU的实际需求，在这种情况下，往往需要调大容器CPU规格，或者优化程序，降低CPU开销。",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 13,
"i": "b302c1a0-a499-4a99-aff2-d460685846ab"
},
"id": "b302c1a0-a499-4a99-aff2-d460685846ab"
},
{
"targets": [
{
"refId": "A",
"expr": "(sum(container_spec_cpu_quota{pod=\"$pod_name\"}/container_spec_cpu_period{pod=\"$pod_name\"}) by (name))"
}
],
"name": "容器CPU Limit",
"description": "容器CPU Limit，2代表容器CPU Limit为2核， 0.2代表容器CPU Limit为0.2核， 没有数据表明没有设置Limit值",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 13,
"i": "cd5e8b99-8a76-4dea-b8b2-3bff825a7f8d"
},
"id": "cd5e8b99-8a76-4dea-b8b2-3bff825a7f8d"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_cpu_load_average_10s{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name)"
}
],
"name": "容器CPU load 10",
"description": "过去10s的CPU负载",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 13,
"i": "408c29b9-2a49-49a2-a98b-3ad7da9c57bd"
},
"id": "408c29b9-2a49-49a2-a98b-3ad7da9c57bd"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_tasks_state{pod=\"$pod_name\", image!~\".*pause.*\", state=\"uninterruptible\"}) by (name)"
}
],
"name": "uninterruptible task 数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 13,
"i": "fe62f7db-4f87-4da1-bfc3-60ed5039e31e"
},
"id": "fe62f7db-4f87-4da1-bfc3-60ed5039e31e"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_tasks_state{pod=\"$pod_name\", image!~\".*pause.*\", state=\"running\"}) by (name)"
}
],
"name": "running task 数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 15,
"i": "7152f66a-d907-4ee9-afd3-a04b12f1019b"
},
"id": "7152f66a-d907-4ee9-afd3-a04b12f1019b"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_tasks_state{pod=\"$pod_name\", image!~\".*pause.*\", state=\"iowaiting\"}) by (name)"
}
],
"name": "ioawaiting task 数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 15,
"i": "d2627af4-5753-4384-880b-84bead73002a"
},
"id": "d2627af4-5753-4384-880b-84bead73002a"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_tasks_state{pod=\"$pod_name\", image!~\".*pause.*\", state=\"sleeping\"}) by (name)"
}
],
"name": "sleeping task 数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 15,
"i": "78801242-2aae-48dc-a3c3-3d0050fac92c"
},
"id": "78801242-2aae-48dc-a3c3-3d0050fac92c"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_tasks_state{pod=\"$pod_name\", image!~\".*pause.*\", state=\"stopped\"}) by (name)"
}
],
"name": "stopped task 数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 15,
"i": "723a4b98-52e8-4284-b193-98a470189415"
},
"id": "723a4b98-52e8-4284-b193-98a470189415"
},
{
"id": "39a8bcef-7b91-4ec4-8644-4e5674267437",
"type": "row",
"name": "内存",
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 22,
"i": "39a8bcef-7b91-4ec4-8644-4e5674267437"
},
"collapsed": true
},
{
"targets": [
{
"refId": "A",
"expr": "((sum(container_memory_usage_bytes{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name)) /(sum(container_spec_memory_limit_bytes{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name)))*100"
}
],
"name": "容器内存使用率",
"description": "(container_memory_rss + container_memory_cache + container_memory_swap + kernel memory)/ (memory limit), 没有数据表明容器配置没有设置mem limit",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "percent",
"max": null
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 0.5,
"gradientMode": "none",
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 23,
"i": "e7b80412-d9b2-4589-8175-abf93a62a524"
},
"id": "e7b80412-d9b2-4589-8175-abf93a62a524"
},
{
"targets": [
{
"refId": "A",
"expr": "(sum(container_memory_usage_bytes{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name)) "
}
],
"name": "容器内存使用",
"description": "container_memory_rss + container_memory_cache + container_memory_swap + kernel memory",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 23,
"i": "4c2e1b54-1adb-480e-a581-1763bec5113f"
},
"id": "4c2e1b54-1adb-480e-a581-1763bec5113f"
},
{
"targets": [
{
"refId": "A",
"expr": "(sum(container_memory_cache{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name))"
}
],
"name": "容器cache使用",
"description": "容器cache 占用大小",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 23,
"i": "429e0214-7b71-4fc0-a771-945363948999"
},
"id": "429e0214-7b71-4fc0-a771-945363948999"
},
{
"targets": [
{
"refId": "A",
"expr": "(sum(container_memory_rss{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name))"
}
],
"name": "容器RSS内存使用",
"description": "容器RSS内存占用大小。",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 23,
"i": "27c1a22c-bdf3-4a8d-bd31-16c66a307f3f"
},
"id": "27c1a22c-bdf3-4a8d-bd31-16c66a307f3f"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_spec_memory_limit_bytes{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name)"
}
],
"name": "容器内存 Limit",
"description": "容器内存Limit配置，0表示没有设置内存 Limit",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 25,
"i": "13173c71-14a9-43a0-9cbe-34c1bb453f2b"
},
"id": "13173c71-14a9-43a0-9cbe-34c1bb453f2b"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_oom_events_total{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name) "
}
],
"name": "容器发生OOM次数",
"custom": {
"textMode": "valueAndName",
"colorMode": "value",
"calc": "lastNotNull",
"colSpan": 1,
"textSize": {}
},
"options": {
"standardOptions": {}
},
"version": "2.0.0",
"type": "stat",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 25,
"i": "d3a7153b-f174-473c-a5ae-1ec8ff4ef318"
},
"id": "d3a7153b-f174-473c-a5ae-1ec8ff4ef318"
},
{
"id": "8d49afd3-d8fa-4714-aaa2-b10e045c8976",
"type": "row",
"name": "磁盘",
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 32,
"i": "8d49afd3-d8fa-4714-aaa2-b10e045c8976"
},
"collapsed": true
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_fs_reads_bytes_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])) by(name)"
}
],
"name": "文件系统读取速率",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 33,
"i": "ef7e1b00-2a77-4a34-a138-5c8207088476"
},
"id": "ef7e1b00-2a77-4a34-a138-5c8207088476"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_fs_writes_bytes_total{pod=\"$pod_name\", image!~\".*pause.*\"}[1m])) by(name)"
}
],
"name": "文件系统写入速率",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 33,
"i": "e55b7db3-2b13-43f9-aa2d-e9a0a929df4b"
},
"id": "e55b7db3-2b13-43f9-aa2d-e9a0a929df4b"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_fs_io_current{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name) "
}
],
"name": "容器I/O",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 33,
"i": "a2d7ba64-fd4d-4713-9295-8663fa72f675"
},
"id": "a2d7ba64-fd4d-4713-9295-8663fa72f675"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_fs_inodes_total{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name) "
}
],
"name": "inode数量",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 33,
"i": "2b4e94ef-c994-4910-b72b-428bf5156a21"
},
"id": "2b4e94ef-c994-4910-b72b-428bf5156a21"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(container_fs_usage_bytes{pod=\"$pod_name\", image!~\".*pause.*\"}) by (name) "
}
],
"name": "容器已使用的文件系统大小",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 35,
"i": "3abade99-8f34-4a1e-bb48-c5ba1e128749"
},
"id": "3abade99-8f34-4a1e-bb48-c5ba1e128749"
},
{
"id": "fde213bc-fe49-431a-9495-2219fcaaa01a",
"type": "row",
"name": "网络",
"layout": {
"h": 1,
"w": 24,
"x": 0,
"y": 42,
"i": "fde213bc-fe49-431a-9495-2219fcaaa01a"
},
"collapsed": true
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_transmit_bytes_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络发送速率",
"description": "容器网络发送的字节数（1 分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 43,
"i": "4e70bc47-1510-41fe-a924-3ffa36d11f0f"
},
"id": "4e70bc47-1510-41fe-a924-3ffa36d11f0f"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_transmit_packets_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络发送数据包",
"description": "网络发送数据包数（1分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 43,
"i": "00ca29e5-7c97-491d-a28f-0f2d66ea47c5"
},
"id": "00ca29e5-7c97-491d-a28f-0f2d66ea47c5"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_transmit_errors_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络发送错误数",
"description": "网络发送错误数（1分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 43,
"i": "8cd76b5f-5bba-421a-9fda-42f406f54ed5"
},
"id": "8cd76b5f-5bba-421a-9fda-42f406f54ed5"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_transmit_packets_dropped_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络发送丢包数",
"description": "网络发送丢包数（1分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 43,
"i": "58aaebbb-b1cf-4089-bcb5-70b03c05318d"
},
"id": "58aaebbb-b1cf-4089-bcb5-70b03c05318d"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_receive_bytes_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络接收速率",
"description": "容器网络接收的字节数（1 分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {
"util": "bytesIEC"
},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 0,
"y": 45,
"i": "120d3540-cce5-4b08-b596-0ec6a5b2d497"
},
"id": "120d3540-cce5-4b08-b596-0ec6a5b2d497"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_receive_packets_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络接收数据包数",
"description": "网络接收数据包数（1分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 6,
"y": 45,
"i": "8b0bac93-8bd0-4452-a200-a9cea74345b3"
},
"id": "8b0bac93-8bd0-4452-a200-a9cea74345b3"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_receive_errors_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络接收错误数",
"description": "网络接收错误数（1分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 12,
"y": 45,
"i": "271d8ff6-2382-488d-b479-83e38b247012"
},
"id": "271d8ff6-2382-488d-b479-83e38b247012"
},
{
"targets": [
{
"refId": "A",
"expr": "sum(rate(container_network_receive_packets_dropped_total{pod=\"$pod_name\"}[1m])) by(name, interface)"
}
],
"name": "网络接收丢包数",
"description": "网络接收丢包数（1分钟内）",
"options": {
"tooltip": {
"mode": "all",
"sort": "none"
},
"legend": {
"displayMode": "hidden"
},
"standardOptions": {},
"thresholds": {}
},
"custom": {
"drawStyle": "lines",
"lineInterpolation": "smooth",
"fillOpacity": 0.5,
"stack": "off"
},
"version": "2.0.0",
"type": "timeseries",
"layout": {
"h": 7,
"w": 6,
"x": 18,
"y": 45,
"i": "e7a0879b-82a9-4518-8c5c-93bc73548401"
},
"id": "e7a0879b-82a9-4518-8c5c-93bc73548401"
}
]
}
}
浙公网安备 33010602011771号