From 9c3d73ef4bd591c9d8bf0b5d6d94f5c5e45ebf85 Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Wed, 25 Mar 2026 04:45:06 +0900 Subject: [PATCH] monitoring: redesign Grafana dashboards --- README.md | 12 +- .../elastickv-cluster-overview.json | 704 ++++++++----- .../dashboards/elastickv-cluster-summary.json | 829 +++++++-------- .../elastickv-dynamodb-summary.json | 974 ------------------ .../dashboards/elastickv-raft-status.json | 651 ++++++++---- 5 files changed, 1337 insertions(+), 1833 deletions(-) delete mode 100644 monitoring/grafana/dashboards/elastickv-dynamodb-summary.json diff --git a/README.md b/README.md index 6db5dba2..dc7f5b0f 100644 --- a/README.md +++ b/README.md @@ -43,16 +43,24 @@ Elastickv now exposes Prometheus metrics on `--metricsAddress` (default: `localh The exported metrics cover: -- DynamoDB-compatible API request rate, success/error split, latency, request/response size, and per-table read/write item counts -- Raft local state, leader identity, current members, commit/applied index, and leader contact lag +- DynamoDB-compatible API request rate, success/system-error/user-error split, latency, in-flight requests, and per-table read/write activity +- Raft local state, leader identity, membership, last-log/commit/applied/snapshot index, FSM backlog, and leader contact lag Provisioned monitoring assets live under: - `monitoring/prometheus/prometheus.yml` - `monitoring/grafana/dashboards/elastickv-cluster-overview.json` +- `monitoring/grafana/dashboards/elastickv-cluster-summary.json` +- `monitoring/grafana/dashboards/elastickv-raft-status.json` - `monitoring/grafana/provisioning/` - `monitoring/docker-compose.yml` +The provisioned dashboards are organized by operator task: + +- `Elastickv Cluster Overview` is the landing page for leader identity, cluster-wide latency/error posture, and per-node Raft health +- `Elastickv Request Health` is the DynamoDB/API drilldown for slow operations, noisy nodes, and hot/erroring tables +- `Elastickv Raft Status` is the control-plane drilldown for membership, node state, index drift, backlog, and leader contact + If you bind `--metricsAddress` to a non-loopback address, `--metricsToken` is required. Prometheus must send the same bearer token, for example: ```yaml diff --git a/monitoring/grafana/dashboards/elastickv-cluster-overview.json b/monitoring/grafana/dashboards/elastickv-cluster-overview.json index 4510596d..f8dd2c26 100644 --- a/monitoring/grafana/dashboards/elastickv-cluster-overview.json +++ b/monitoring/grafana/dashboards/elastickv-cluster-overview.json @@ -15,9 +15,10 @@ } ] }, + "description": "Operator landing page for elastickv: leader, latency, error rate, and per-node Raft health.", "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [], "panels": [ @@ -33,7 +34,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] }, @@ -43,7 +44,7 @@ }, "gridPos": { "h": 4, - "w": 6, + "w": 4, "x": 0, "y": 0 }, @@ -66,14 +67,13 @@ { "datasource": null, "editorMode": "code", - "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\"}[5m]))", - "instant": false, + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\"}[5m]))", "legendFormat": "requests/s", "range": true, "refId": "A" } ], - "title": "Request Rate", + "title": "Requests / s", "type": "stat" }, { @@ -88,26 +88,26 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "orange", - "value": 0.1 + "value": 0.01 }, { "color": "red", - "value": 1 + "value": 0.05 } ] }, - "unit": "reqps" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, - "w": 6, - "x": 6, + "w": 4, + "x": 4, "y": 0 }, "id": 2, @@ -129,14 +129,13 @@ { "datasource": null, "editorMode": "code", - "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",outcome!=\"success\"}[5m]))", - "instant": false, - "legendFormat": "errors/s", + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"system_error\"}[5m])) / clamp_min(sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\"}[5m])), 0.001)", + "legendFormat": "system error ratio", "range": true, "refId": "A" } ], - "title": "Error Rate", + "title": "System Error Ratio (5m)", "type": "stat" }, { @@ -151,7 +150,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "orange", @@ -169,8 +168,8 @@ }, "gridPos": { "h": 4, - "w": 6, - "x": 12, + "w": 4, + "x": 8, "y": 0 }, "id": 3, @@ -192,14 +191,58 @@ { "datasource": null, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\"}[5m])))", - "instant": false, - "legendFormat": "p95", + "expr": "histogram_quantile(0.95, sum by (le) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"success\"}[5m])))", + "legendFormat": "p95 success", + "range": true, + "refId": "A" + } + ], + "title": "Success P95 Latency", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"conditional_check_failed\"}[5m]))", + "legendFormat": "conditional check failed/s", "range": true, "refId": "A" } ], - "title": "P95 Latency", + "title": "Conditional Check Failed / s", "type": "stat" }, { @@ -214,7 +257,15 @@ "steps": [ { "color": "green", - "value": null + "value": 0 + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 100 } ] }, @@ -224,11 +275,11 @@ }, "gridPos": { "h": 4, - "w": 6, - "x": 18, + "w": 4, + "x": 16, "y": 0 }, - "id": 4, + "id": 5, "options": { "colorMode": "value", "graphMode": "area", @@ -247,14 +298,13 @@ { "datasource": null, "editorMode": "code", - "expr": "sum(elastickv_dynamodb_inflight_requests{job=\"elastickv\"})", - "instant": false, - "legendFormat": "in-flight", + "expr": "max(elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "max commit-applied", "range": true, "refId": "A" } ], - "title": "In-flight Requests", + "title": "Max Commit-Applied Gap", "type": "stat" }, { @@ -262,86 +312,157 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, - "unit": "reqps" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 0.2 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 + "h": 4, + "w": 4, + "x": 20, + "y": 0 }, - "id": 5, + "id": 6, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "textMode": "auto" }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\"}[5m]))", - "legendFormat": "{{operation}}", + "expr": "max(clamp_min(elastickv_raft_last_contact_seconds{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"}, 0))", + "legendFormat": "max last contact", "range": true, "refId": "A" } ], - "title": "Requests by Operation", - "type": "timeseries" + "title": "Max Last Leader Contact", + "type": "stat" }, { "datasource": null, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } }, - "unit": "reqps" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, + "h": 7, + "w": 8, + "x": 0, "y": 4 }, - "id": 6, + "id": 7, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "showHeader": true }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum by (outcome) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",outcome!=\"success\"}[5m]))", - "legendFormat": "{{outcome}}", - "range": true, + "expr": "max by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "count by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "B" } ], - "title": "Error Split by Outcome", - "type": "timeseries" + "title": "Leader by Group", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Metric": true, + "Time": true, + "Value": true, + "Value #A": true + }, + "indexByName": { + "group": 0, + "leader_id": 1, + "leader_address": 2, + "Value #B": 3 + }, + "renameByName": { + "group": "Raft Group", + "leader_address": "Leader Address", + "leader_id": "Leader ID", + "Value #B": "Observer Count" + } + } + } + ], + "type": "table" }, { "datasource": null, @@ -350,20 +471,25 @@ "color": { "mode": "palette-classic" }, - "unit": "ops" + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 + "h": 7, + "w": 8, + "x": 8, + "y": 4 }, - "id": 7, + "id": 8, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -377,29 +503,13 @@ { "datasource": null, "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_returned_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "returned {{table}}", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\"}[5m]))", + "legendFormat": "{{operation}}", "range": true, "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_scanned_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "scanned {{table}}", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_written_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "written {{table}}", - "range": true, - "refId": "C" } ], - "title": "Per-table Item Activity", + "title": "Request Rate by Operation", "type": "timeseries" }, { @@ -409,20 +519,25 @@ "color": { "mode": "palette-classic" }, - "unit": "short" + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 + "h": 7, + "w": 8, + "x": 16, + "y": 4 }, - "id": 8, + "id": 9, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -436,21 +551,13 @@ { "datasource": null, "editorMode": "code", - "expr": "max by (group, node_id) (elastickv_raft_commit_index{job=\"elastickv\"})", - "legendFormat": "commit g{{group}} {{node_id}}", + "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"success\"}[5m])))", + "legendFormat": "{{operation}}", "range": true, "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "max by (group, node_id) (elastickv_raft_applied_index{job=\"elastickv\"})", - "legendFormat": "applied g{{group}} {{node_id}}", - "range": true, - "refId": "B" } ], - "title": "Raft Index Progress", + "title": "Success P95 Latency by Operation", "type": "timeseries" }, { @@ -469,7 +576,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -477,14 +584,13 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, + "h": 10, + "w": 24, "x": 0, - "y": 20 + "y": 11 }, - "id": 9, + "id": 10, "options": { - "cellHeight": "sm", "footer": { "countRows": false, "fields": "", @@ -499,15 +605,69 @@ { "datasource": null, "editorMode": "code", - "expr": "max by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\"} * on(group, node_id, node_address) (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", + "expr": "max by (group, node_id, node_address, state) (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} == 1)", "format": "table", "instant": true, - "legendFormat": "group {{group}} leader {{leader_id}}", - "range": false, "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "C" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "E" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "F" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (clamp_min(elastickv_raft_last_contact_seconds{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"}, 0))", + "format": "table", + "instant": true, + "refId": "G" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_fsm_pending{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "H" } ], - "title": "Current Leader per Group", + "title": "Node Health Snapshot", "transformations": [ { "id": "labelsToFields", @@ -528,13 +688,29 @@ }, "indexByName": { "group": 0, - "leader_id": 1, - "leader_address": 2 + "node_id": 1, + "node_address": 2, + "state": 3, + "Value #B": 4, + "Value #C": 5, + "Value #D": 6, + "Value #E": 7, + "Value #F": 8, + "Value #G": 9, + "Value #H": 10 }, "renameByName": { "group": "Raft Group", - "leader_address": "Leader Address", - "leader_id": "Leader ID" + "node_address": "Node Address", + "node_id": "Node ID", + "state": "Local State", + "Value #B": "Last Log Index", + "Value #C": "Commit Index", + "Value #D": "Applied Index", + "Value #E": "Last Log-Applied", + "Value #F": "Commit-Applied", + "Value #G": "Last Contact (s)", + "Value #H": "FSM Pending" } } } @@ -545,214 +721,188 @@ "datasource": null, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - } + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 8, - "y": 20 + "w": 12, + "x": 0, + "y": 21 }, - "id": 10, + "id": 11, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showHeader": true + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, member_id, member_address, suffrage) (elastickv_raft_member_present{job=\"elastickv\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", - "format": "table", - "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", - "range": false, + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"system_error\"}[5m]))", + "legendFormat": "system_error {{operation}}", + "range": true, "refId": "A" - } - ], - "title": "Current Members", - "transformations": [ - { - "id": "labelsToFields", - "options": {} }, { - "id": "merge", - "options": {} + "datasource": null, + "editorMode": "code", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"user_error\"}[5m]))", + "legendFormat": "user_error {{operation}}", + "range": true, + "refId": "B" }, { - "id": "organize", - "options": { - "excludeByName": { - "Metric": true, - "Time": true, - "Value": true, - "Value #A": true - }, - "indexByName": { - "group": 0, - "member_id": 1, - "member_address": 2, - "suffrage": 3 - }, - "renameByName": { - "group": "Raft Group", - "member_address": "Member Address", - "member_id": "Member ID", - "suffrage": "Suffrage" - } - } + "datasource": null, + "editorMode": "code", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",outcome=\"conditional_check_failed\"}[5m]))", + "legendFormat": "conditional {{operation}}", + "range": true, + "refId": "C" } ], - "type": "table" + "title": "System / User / Conditional Errors by Operation", + "type": "timeseries" }, { "datasource": null, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - } + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" }, - "mappings": [ - { - "options": { - "0": { - "text": "Follower" - }, - "1": { - "text": "Leader" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 20 + "w": 12, + "x": 12, + "y": 21 }, - "id": 11, + "id": 12, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showHeader": true + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, member_id, member_address) (elastickv_raft_member_is_leader{job=\"elastickv\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", - "format": "table", - "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", - "range": false, + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "g{{group}} {{node_id}}", + "range": true, "refId": "A" } ], - "title": "Leader Flag per Member", - "transformations": [ - { - "id": "labelsToFields", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Metric": true, - "Time": true - }, - "indexByName": { - "group": 0, - "member_id": 1, - "member_address": 2, - "Value": 3, - "Value #A": 3 - }, - "renameByName": { - "Value": "Leader Flag", - "Value #A": "Leader Flag", - "group": "Raft Group", - "member_address": "Member Address", - "member_id": "Member ID" - } - } - } - ], - "type": "table" + "title": "Commit-Applied Gap by Node", + "type": "timeseries" } ], "refresh": "10s", "schemaVersion": 41, - "style": "dark", "tags": [ "elastickv", + "overview", "raft", "dynamodb" ], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(elastickv_raft_term{job=\"elastickv\"}, group)", + "includeAll": true, + "label": "Raft Group", + "multi": true, + "name": "group", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(elastickv_raft_term{job=\"elastickv\"}, group)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\"}, node_id)", + "includeAll": true, + "label": "Node ID", + "multi": true, + "name": "node_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\"}, node_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, - "timezone": "", + "timezone": "browser", "title": "Elastickv Cluster Overview", "uid": "elastickv-cluster", - "version": 1, - "weekStart": "" + "version": 2 } diff --git a/monitoring/grafana/dashboards/elastickv-cluster-summary.json b/monitoring/grafana/dashboards/elastickv-cluster-summary.json index de0c99bc..b6fea4ee 100644 --- a/monitoring/grafana/dashboards/elastickv-cluster-summary.json +++ b/monitoring/grafana/dashboards/elastickv-cluster-summary.json @@ -15,10 +15,10 @@ } ] }, - "description": "Compact overview of DynamoDB request activity and core Raft state for elastickv.", + "description": "DynamoDB-compatible request health for elastickv: throughput, latency, errors, tables, and noisy nodes.", "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [], "panels": [ @@ -29,7 +29,6 @@ "color": { "mode": "thresholds" }, - "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -45,7 +44,7 @@ }, "gridPos": { "h": 4, - "w": 6, + "w": 4, "x": 0, "y": 0 }, @@ -53,33 +52,25 @@ "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\"}[5m]))", - "instant": false, + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\"}[5m]))", "legendFormat": "requests/s", "range": true, "refId": "A" } ], - "title": "Request Rate", + "title": "Requests / s", "type": "stat" }, { @@ -89,65 +80,158 @@ "color": { "mode": "thresholds" }, - "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": 0 }, { "color": "orange", - "value": 0.1 + "value": 0.95 }, { - "color": "red", - "value": 1 + "color": "green", + "value": 0.99 } ] }, - "unit": "reqps" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, - "w": 6, - "x": 6, + "w": 4, + "x": 4, "y": 0 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false + } + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"success\"}[5m])) / clamp_min(sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\"}[5m])), 0.001)", + "legendFormat": "success rate", + "range": true, + "refId": "A" + } + ], + "title": "Success Rate", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"system_error\"}[5m]))", + "legendFormat": "system errors/s", + "range": true, + "refId": "A" + } + ], + "title": "System Errors / s", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",outcome!=\"success\"}[5m]))", - "instant": false, - "legendFormat": "errors/s", + "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"user_error\"}[5m]))", + "legendFormat": "user errors/s", "range": true, "refId": "A" } ], - "title": "Error Rate", + "title": "User Errors / s", "type": "stat" }, { @@ -157,7 +241,6 @@ "color": { "mode": "thresholds" }, - "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -181,41 +264,33 @@ }, "gridPos": { "h": 4, - "w": 6, - "x": 12, + "w": 4, + "x": 16, "y": 0 }, - "id": 3, + "id": 5, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\"}[5m])))", - "instant": false, - "legendFormat": "p95", + "expr": "histogram_quantile(0.95, sum by (le) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"success\"}[5m])))", + "legendFormat": "p95 success", "range": true, "refId": "A" } ], - "title": "P95 Latency", + "title": "Success P95 Latency", "type": "stat" }, { @@ -225,7 +300,6 @@ "color": { "mode": "thresholds" }, - "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -241,41 +315,33 @@ }, "gridPos": { "h": 4, - "w": 6, - "x": 18, + "w": 4, + "x": 20, "y": 0 }, - "id": 4, + "id": 6, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum(elastickv_dynamodb_inflight_requests{job=\"elastickv\"})", - "instant": false, - "legendFormat": "in-flight", + "expr": "sum(elastickv_dynamodb_inflight_requests{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\"})", + "legendFormat": "inflight", "range": true, "refId": "A" } ], - "title": "In-flight Requests", + "title": "Inflight Requests", "type": "stat" }, { @@ -286,51 +352,10 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto" }, "unit": "reqps" }, @@ -342,32 +367,29 @@ "x": 0, "y": 4 }, - "id": 5, + "id": 7, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\"}[5m]))", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\"}[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], - "title": "Requests by Operation", + "title": "Request Rate by Operation", "type": "timeseries" }, { @@ -378,53 +400,12 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "unit": "s" }, "overrides": [] }, @@ -434,32 +415,45 @@ "x": 12, "y": 4 }, - "id": 6, + "id": 8, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum by (outcome) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",outcome!=\"success\"}[5m]))", - "legendFormat": "{{outcome}}", + "expr": "histogram_quantile(0.50, sum by (le, operation) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"success\"}[5m])))", + "legendFormat": "p50 {{operation}}", "range": true, "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"success\"}[5m])))", + "legendFormat": "p95 {{operation}}", + "range": true, + "refId": "B" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, operation) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"success\"}[5m])))", + "legendFormat": "p99 {{operation}}", + "range": true, + "refId": "C" } ], - "title": "Error Split by Outcome", + "title": "Latency P50 / P95 / P99 by Operation", "type": "timeseries" }, { @@ -470,53 +464,12 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" + "unit": "reqps" }, "overrides": [] }, @@ -526,48 +479,45 @@ "x": 0, "y": 12 }, - "id": 7, + "id": 9, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_returned_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "returned {{table}}", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"system_error\"}[5m]))", + "legendFormat": "system_error {{operation}}", "range": true, "refId": "A" }, { "datasource": null, "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_scanned_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "scanned {{table}}", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"user_error\"}[5m]))", + "legendFormat": "user_error {{operation}}", "range": true, "refId": "B" }, { "datasource": null, "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_written_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "written {{table}}", + "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"conditional_check_failed\"}[5m]))", + "legendFormat": "conditional {{operation}}", "range": true, "refId": "C" } ], - "title": "Per-table Item Activity", + "title": "System / User / Conditional Errors by Operation", "type": "timeseries" }, { @@ -578,53 +528,12 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "unit": "reqps" }, "overrides": [] }, @@ -634,40 +543,125 @@ "x": 12, "y": 12 }, - "id": 8, + "id": 10, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, node_id) (elastickv_raft_commit_index{job=\"elastickv\"})", - "legendFormat": "commit g{{group}} {{node_id}}", + "expr": "sum by (node_id, node_address) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\"}[5m]))", + "legendFormat": "{{node_id}} {{node_address}}", "range": true, "refId": "A" + } + ], + "title": "Request Rate by Node", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, node_id) (elastickv_raft_applied_index{job=\"elastickv\"})", - "legendFormat": "applied g{{group}} {{node_id}}", + "expr": "histogram_quantile(0.95, sum by (le, node_id, node_address) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",outcome=\"success\"}[5m])))", + "legendFormat": "{{node_id}} {{node_address}}", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Raft Index Progress", + "title": "Success P95 Latency by Node", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum by (operation) (elastickv_dynamodb_inflight_requests{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\"})", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Inflight by Operation", "type": "timeseries" }, { @@ -678,55 +672,32 @@ "align": "auto", "cellOptions": { "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] + } } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 8, "x": 0, - "y": 20 + "y": 28 }, - "id": 9, + "id": 13, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, "showHeader": true }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\"} * on(group, node_id, node_address) (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", + "expr": "topk(10, sum by (table) (rate(elastickv_dynamodb_table_requests_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m])))", "format": "table", "instant": true, - "legendFormat": "group {{group}} leader {{leader_id}}", - "range": false, "refId": "A" } ], - "title": "Current Leader per Group", + "title": "Top Tables by Request Rate", "transformations": [ { "id": "labelsToFields", @@ -742,18 +713,15 @@ "excludeByName": { "Metric": true, "Time": true, - "Value": true, - "Value #A": true + "Value": true }, "indexByName": { - "group": 0, - "leader_id": 1, - "leader_address": 2 + "table": 0, + "Value #A": 1 }, "renameByName": { - "group": "Raft Group", - "leader_address": "Leader Address", - "leader_id": "Leader ID" + "table": "Table", + "Value #A": "Requests / s" } } } @@ -768,55 +736,48 @@ "align": "auto", "cellOptions": { "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] + } } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 8, "x": 8, - "y": 20 + "y": 28 }, - "id": 10, + "id": 14, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, "showHeader": true }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, member_id, member_address, suffrage) (elastickv_raft_member_present{job=\"elastickv\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", + "expr": "sum by (table) (rate(elastickv_dynamodb_system_errors_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m]))", "format": "table", "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", - "range": false, "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "sum by (table) (rate(elastickv_dynamodb_user_errors_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m]))", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "sum by (table) (rate(elastickv_dynamodb_conditional_check_failed_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m]))", + "format": "table", + "instant": true, + "refId": "C" } ], - "title": "Current Members", + "title": "Tables by System / User / Conditional Error Rate", "transformations": [ { "id": "labelsToFields", @@ -832,20 +793,19 @@ "excludeByName": { "Metric": true, "Time": true, - "Value": true, - "Value #A": true + "Value": true }, "indexByName": { - "group": 0, - "member_id": 1, - "member_address": 2, - "suffrage": 3 + "table": 0, + "Value #A": 1, + "Value #B": 2, + "Value #C": 3 }, "renameByName": { - "group": "Raft Group", - "member_address": "Member Address", - "member_id": "Member ID", - "suffrage": "Suffrage" + "table": "Table", + "Value #A": "System Errors / s", + "Value #B": "User Errors / s", + "Value #C": "Conditional / s" } } } @@ -860,67 +820,48 @@ "align": "auto", "cellOptions": { "type": "auto" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "0": { - "text": "Follower" - }, - "1": { - "text": "Leader" - } - }, - "type": "value" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 8, "x": 16, - "y": 20 + "y": 28 }, - "id": 11, + "id": 15, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, "showHeader": true }, - "pluginVersion": "12.1.1", "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, member_id, member_address) (elastickv_raft_member_is_leader{job=\"elastickv\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", + "expr": "sum by (table) (rate(elastickv_dynamodb_returned_items_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m]))", "format": "table", "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", - "range": false, "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "sum by (table) (rate(elastickv_dynamodb_scanned_items_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m]))", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "sum by (table) (rate(elastickv_dynamodb_written_items_total{job=\"elastickv\",node_id=~\"$node_id\",operation=~\"$operation\",table=~\"$table\"}[5m]))", + "format": "table", + "instant": true, + "refId": "C" } ], - "title": "Leader Flag per Member", + "title": "Returned / Scanned / Written by Table", "transformations": [ { "id": "labelsToFields", @@ -935,21 +876,20 @@ "options": { "excludeByName": { "Metric": true, - "Time": true + "Time": true, + "Value": true }, "indexByName": { - "group": 0, - "member_id": 1, - "member_address": 2, - "Value": 3, - "Value #A": 3 + "table": 0, + "Value #A": 1, + "Value #B": 2, + "Value #C": 3 }, "renameByName": { - "Value": "Leader Flag", - "Value #A": "Leader Flag", - "group": "Raft Group", - "member_address": "Member Address", - "member_id": "Member ID" + "table": "Table", + "Value #A": "Returned / s", + "Value #B": "Scanned / s", + "Value #C": "Written / s" } } } @@ -957,25 +897,106 @@ "type": "table" } ], - "preload": false, "refresh": "10s", "schemaVersion": 41, "tags": [ "elastickv", - "raft", "dynamodb", + "requests", "summary" ], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(elastickv_dynamodb_requests_total{job=\"elastickv\"}, node_id)", + "includeAll": true, + "label": "Node ID", + "multi": true, + "name": "node_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(elastickv_dynamodb_requests_total{job=\"elastickv\"}, node_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(elastickv_dynamodb_requests_total{job=\"elastickv\"}, operation)", + "includeAll": true, + "label": "Operation", + "multi": true, + "name": "operation", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(elastickv_dynamodb_requests_total{job=\"elastickv\"}, operation)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(elastickv_dynamodb_table_requests_total{job=\"elastickv\"}, table)", + "includeAll": true, + "label": "Table", + "multi": true, + "name": "table", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(elastickv_dynamodb_table_requests_total{job=\"elastickv\"}, table)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] }, "time": { - "from": "now-2d", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", - "title": "Elastickv Cluster Summary", + "title": "Elastickv Request Health", "uid": "elastickv-cluster-summary", - "version": 1 + "version": 2 } diff --git a/monitoring/grafana/dashboards/elastickv-dynamodb-summary.json b/monitoring/grafana/dashboards/elastickv-dynamodb-summary.json deleted file mode 100644 index 4bcd399c..00000000 --- a/monitoring/grafana/dashboards/elastickv-dynamodb-summary.json +++ /dev/null @@ -1,974 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "DynamoDB request activity with leader-scoped Raft membership context for elastickv.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\"}[5m]))", - "instant": false, - "legendFormat": "requests/s", - "range": true, - "refId": "A" - } - ], - "title": "Request Rate", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 0.1 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(elastickv_dynamodb_requests_total{job=\"elastickv\",outcome!=\"success\"}[5m]))", - "instant": false, - "legendFormat": "errors/s", - "range": true, - "refId": "A" - } - ], - "title": "Error Rate", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 0.05 - }, - { - "color": "red", - "value": 0.25 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le) (rate(elastickv_dynamodb_request_duration_seconds_bucket{job=\"elastickv\"}[5m])))", - "instant": false, - "legendFormat": "p95", - "range": true, - "refId": "A" - } - ], - "title": "P95 Latency", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(elastickv_dynamodb_inflight_requests{job=\"elastickv\"})", - "instant": false, - "legendFormat": "in-flight", - "range": true, - "refId": "A" - } - ], - "title": "In-flight Requests", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (operation) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\"}[5m]))", - "legendFormat": "{{operation}}", - "range": true, - "refId": "A" - } - ], - "title": "Requests by Operation", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (outcome) (rate(elastickv_dynamodb_requests_total{job=\"elastickv\",outcome!=\"success\"}[5m]))", - "legendFormat": "{{outcome}}", - "range": true, - "refId": "A" - } - ], - "title": "Error Split by Outcome", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_returned_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "returned {{table}}", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_scanned_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "scanned {{table}}", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum by (table) (rate(elastickv_dynamodb_written_items_total{job=\"elastickv\"}[5m]))", - "legendFormat": "written {{table}}", - "range": true, - "refId": "C" - } - ], - "title": "Per-table Item Activity", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "max by (group, node_id) (elastickv_raft_commit_index{job=\"elastickv\"})", - "legendFormat": "commit g{{group}} {{node_id}}", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "max by (group, node_id) (elastickv_raft_applied_index{job=\"elastickv\"})", - "legendFormat": "applied g{{group}} {{node_id}}", - "range": true, - "refId": "B" - } - ], - "title": "Raft Index Progress", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 9, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "max by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\"} * on(group, node_id, node_address) (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", - "format": "table", - "instant": true, - "legendFormat": "group {{group}} leader {{leader_id}}", - "range": false, - "refId": "A" - } - ], - "title": "Current Leader per Group", - "transformations": [ - { - "id": "labelsToFields", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Metric": true, - "Time": true, - "Value": true, - "Value #A": true - }, - "indexByName": { - "group": 0, - "leader_address": 2, - "leader_id": 1 - }, - "renameByName": { - "group": "Raft Group", - "leader_address": "Leader Address", - "leader_id": "Leader ID" - } - } - } - ], - "type": "table" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 10, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "max by (group, member_id, member_address, suffrage) (elastickv_raft_member_present{job=\"elastickv\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", - "format": "table", - "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", - "range": false, - "refId": "A" - } - ], - "title": "Current Members", - "transformations": [ - { - "id": "labelsToFields", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Metric": true, - "Time": true, - "Value": true, - "Value #A": true - }, - "indexByName": { - "group": 0, - "member_address": 2, - "member_id": 1, - "suffrage": 3 - }, - "renameByName": { - "group": "Raft Group", - "member_address": "Member Address", - "member_id": "Member ID", - "suffrage": "Suffrage" - } - } - } - ], - "type": "table" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - } - }, - "mappings": [ - { - "options": { - "0": { - "text": "Follower" - }, - "1": { - "text": "Leader" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 11, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "max by (group, member_id, member_address) (elastickv_raft_member_is_leader{job=\"elastickv\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",state=\"leader\"} == 1))", - "format": "table", - "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", - "range": false, - "refId": "A" - } - ], - "title": "Leader Flag per Member", - "transformations": [ - { - "id": "labelsToFields", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Metric": true, - "Time": true - }, - "indexByName": { - "Value": 3, - "Value #A": 3, - "group": 0, - "member_address": 2, - "member_id": 1 - }, - "renameByName": { - "Value": "Leader Flag", - "Value #A": "Leader Flag", - "group": "Raft Group", - "member_address": "Member Address", - "member_id": "Member ID" - } - } - } - ], - "type": "table" - } - ], - "preload": false, - "refresh": "10s", - "schemaVersion": 41, - "tags": [ - "dynamodb", - "elastickv", - "raft", - "summary" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Elastickv DynamoDB Summary", - "uid": "elastickv-dynamodb-summary", - "version": 6 -} diff --git a/monitoring/grafana/dashboards/elastickv-raft-status.json b/monitoring/grafana/dashboards/elastickv-raft-status.json index b6803991..f7b50f1e 100644 --- a/monitoring/grafana/dashboards/elastickv-raft-status.json +++ b/monitoring/grafana/dashboards/elastickv-raft-status.json @@ -15,25 +15,13 @@ } ] }, - "description": "Raft consensus status for elastickv: term, leadership, index progress, replication lag, and FSM backpressure.", + "description": "Raft control-plane status for elastickv: leader, membership, node state, index drift, backlog, and leader contact.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "title": "Cluster Overview", - "type": "row" - }, { "datasource": null, "fieldConfig": { @@ -46,7 +34,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] }, @@ -58,35 +46,31 @@ "h": 4, "w": 6, "x": 0, - "y": 1 + "y": 0 }, - "id": 2, + "id": 1, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "textMode": "auto" + } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_term{job=\"elastickv\",group=~\"$group\"} * on(group, node_id, node_address) (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",state=\"leader\"} == 1))", - "instant": false, + "expr": "max by (group) (elastickv_raft_term{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", "legendFormat": "{{group}}", "range": true, "refId": "A" } ], - "title": "Raft Term", + "title": "Current Term", "type": "stat" }, { @@ -101,7 +85,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] }, @@ -113,35 +97,31 @@ "h": 4, "w": 6, "x": 6, - "y": 1 + "y": 0 }, - "id": 3, + "id": 2, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "textMode": "auto" + } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "clamp_min(count by (group) (elastickv_raft_member_present{job=\"elastickv\",group=~\"$group\",suffrage=\"voter\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",state=\"leader\"} == 1)) - 1, 0)", - "instant": false, + "expr": "max by (group) (elastickv_raft_members{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", "legendFormat": "{{group}}", "range": true, "refId": "A" } ], - "title": "Other Voting Peers", + "title": "Member Count", "type": "stat" }, { @@ -156,11 +136,19 @@ "steps": [ { "color": "green", - "value": null + "value": 0 + }, + { + "color": "orange", + "value": 0.2 + }, + { + "color": "red", + "value": 1 } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, @@ -168,35 +156,31 @@ "h": 4, "w": 6, "x": 12, - "y": 1 + "y": 0 }, - "id": 4, + "id": 3, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "textMode": "auto" + } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "count by (group) (elastickv_raft_member_present{job=\"elastickv\",group=~\"$group\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",state=\"leader\"} == 1))", - "instant": false, + "expr": "max by (group) (clamp_min(elastickv_raft_last_contact_seconds{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"}, 0))", "legendFormat": "{{group}}", "range": true, "refId": "A" } ], - "title": "Config Members", + "title": "Max Follower Last Contact", "type": "stat" }, { @@ -211,10 +195,10 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { - "color": "yellow", + "color": "orange", "value": 10 }, { @@ -231,85 +215,75 @@ "h": 4, "w": 6, "x": 18, - "y": 1 + "y": 0 }, - "id": 5, + "id": 4, "options": { "colorMode": "value", "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "textMode": "auto" + } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_fsm_pending{job=\"elastickv\",group=~\"$group\"})", - "instant": false, + "expr": "max by (group) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", "legendFormat": "{{group}}", "range": true, "refId": "A" } ], - "title": "Max FSM Pending", + "title": "Max Commit-Applied Gap", "type": "stat" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 6, - "title": "Leader & State", - "type": "row" - }, { "datasource": null, "fieldConfig": { - "defaults": {}, + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 7, + "w": 8, "x": 0, - "y": 6 + "y": 4 }, - "id": 7, + "id": 5, "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, "showHeader": true }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\",group=~\"$group\"} * on(group, node_id, node_address) (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",state=\"leader\"} == 1))", + "expr": "max by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", "format": "table", "instant": true, - "legendFormat": "group {{group}} leader {{leader_id}}", "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "count by (group, leader_id, leader_address) (elastickv_raft_leader_identity{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "B" } ], - "title": "Current Leader per Group", + "title": "Current Leader", "transformations": [ { "id": "labelsToFields", @@ -331,12 +305,14 @@ "indexByName": { "group": 0, "leader_id": 1, - "leader_address": 2 + "leader_address": 2, + "Value #B": 3 }, "renameByName": { "group": "Raft Group", "leader_address": "Leader Address", - "leader_id": "Leader ID" + "leader_id": "Leader ID", + "Value #B": "Observer Count" } } } @@ -346,35 +322,33 @@ { "datasource": null, "fieldConfig": { - "defaults": {}, + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 + "h": 7, + "w": 8, + "x": 8, + "y": 4 }, - "id": 8, + "id": 6, "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, "showHeader": true }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, node_id, node_address, state) (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\"} == 1)", + "expr": "max by (group, node_id, node_address, state) (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} == 1)", "format": "table", "instant": true, - "legendFormat": "group {{group}} node {{node_id}} {{state}}", "refId": "A" } ], @@ -417,39 +391,72 @@ { "datasource": null, "fieldConfig": { - "defaults": {}, - "overrides": [] + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Leader Flag" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "follower" + }, + "1": { + "index": 1, + "text": "leader" + } + }, + "type": "value" + } + ] + } + ] + } + ] }, "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 14 + "h": 7, + "w": 8, + "x": 16, + "y": 4 }, - "id": 9, + "id": 7, "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, "showHeader": true }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group, member_id, member_address, suffrage) (elastickv_raft_member_present{job=\"elastickv\",group=~\"$group\"} * on(group, node_id, node_address) group_left() (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",state=\"leader\"} == 1))", + "expr": "max by (group, member_id, member_address, suffrage) (elastickv_raft_member_present{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", "format": "table", "instant": true, - "legendFormat": "group {{group}} member {{member_id}}", "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, member_id, member_address) (elastickv_raft_member_is_leader{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "B" } ], - "title": "Cluster Members", + "title": "Membership", "transformations": [ { "id": "labelsToFields", @@ -472,13 +479,15 @@ "group": 0, "member_id": 1, "member_address": 2, - "suffrage": 3 + "suffrage": 3, + "Value #B": 4 }, "renameByName": { "group": "Raft Group", "member_address": "Member Address", "member_id": "Member ID", - "suffrage": "Suffrage" + "suffrage": "Suffrage", + "Value #B": "Leader Flag" } } } @@ -486,16 +495,176 @@ "type": "table" }, { - "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Last Contact (s)" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "-1": { + "index": 0, + "text": "unknown" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, "gridPos": { - "h": 1, + "h": 9, "w": 24, "x": 0, - "y": 22 + "y": 11 }, - "id": 10, - "title": "Index Progress", - "type": "row" + "id": 8, + "options": { + "showHeader": true + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address, state) (elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} == 1)", + "format": "table", + "instant": true, + "refId": "A" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "C" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_snapshot_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "E" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "F" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "G" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_contact_seconds{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "H" + }, + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_fsm_pending{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "format": "table", + "instant": true, + "refId": "I" + } + ], + "title": "Per-Node Index Snapshot", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Metric": true, + "Time": true, + "Value": true, + "Value #A": true + }, + "indexByName": { + "group": 0, + "node_id": 1, + "node_address": 2, + "state": 3, + "Value #B": 4, + "Value #C": 5, + "Value #D": 6, + "Value #E": 7, + "Value #F": 8, + "Value #G": 9, + "Value #H": 10, + "Value #I": 11 + }, + "renameByName": { + "group": "Raft Group", + "node_address": "Node Address", + "node_id": "Node ID", + "state": "Local State", + "Value #B": "Last Log Index", + "Value #C": "Commit Index", + "Value #D": "Applied Index", + "Value #E": "Snapshot Index", + "Value #F": "Last Log-Applied", + "Value #G": "Commit-Applied", + "Value #H": "Last Contact (s)", + "Value #I": "FSM Pending" + } + } + } + ], + "type": "table" }, { "datasource": null, @@ -504,6 +673,12 @@ "color": { "mode": "palette-classic" }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, "unit": "short" }, "overrides": [] @@ -512,73 +687,104 @@ "h": 8, "w": 24, "x": 0, - "y": 23 + "y": 20 }, - "id": 11, + "id": 9, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\"})", - "instant": false, - "legendFormat": "last log - {{group}}", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "last_log g{{group}} {{node_id}}", "range": true, "refId": "A" }, { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\"})", - "instant": false, - "legendFormat": "commit - {{group}}", + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "commit g{{group}} {{node_id}}", "range": true, "refId": "B" }, { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\"})", - "instant": false, - "legendFormat": "applied - {{group}}", + "expr": "max by (group, node_id, node_address) (elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "applied g{{group}} {{node_id}}", "range": true, "refId": "C" }, { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_last_snapshot_index{job=\"elastickv\",group=~\"$group\"})", - "instant": false, - "legendFormat": "snapshot - {{group}}", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_snapshot_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "snapshot g{{group}} {{node_id}}", "range": true, "refId": "D" } ], - "title": "Index Progress", + "title": "Last Log / Commit / Applied / Snapshot by Node", "type": "timeseries" }, { - "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 31 + "y": 28 }, - "id": 12, - "title": "Replication Health", - "type": "row" + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "g{{group}} {{node_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Log-Applied Backlog by Node", + "type": "timeseries" }, { "datasource": null, @@ -587,6 +793,12 @@ "color": { "mode": "palette-classic" }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, "unit": "short" }, "overrides": [] @@ -594,34 +806,32 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 32 + "x": 12, + "y": 28 }, - "id": 13, + "id": 11, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_last_log_index{job=\"elastickv\",group=~\"$group\"}) - max by (group) (elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\"})", - "instant": false, - "legendFormat": "lag - {{group}}", + "expr": "max by (group, node_id, node_address) (elastickv_raft_commit_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"} - on(group, node_id, node_address) elastickv_raft_applied_index{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "g{{group}} {{node_id}}", "range": true, "refId": "A" } ], - "title": "Replication Lag (unapplied entries)", + "title": "Commit-Applied Backlog by Node", "type": "timeseries" }, { @@ -631,79 +841,168 @@ "color": { "mode": "palette-classic" }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "max by (group, node_id, node_address) (clamp_min(elastickv_raft_last_contact_seconds{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"}, 0))", + "legendFormat": "g{{group}} {{node_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Contact by Node", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto" + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 36 }, - "id": 14, + "id": 13, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, "targets": [ { "datasource": null, "editorMode": "code", - "expr": "max by (group) (elastickv_raft_last_contact_seconds{job=\"elastickv\",group=~\"$group\"})", - "instant": false, - "legendFormat": "{{group}}", + "expr": "max by (group, node_id, node_address) (elastickv_raft_fsm_pending{job=\"elastickv\",group=~\"$group\",node_id=~\"$node_id\"})", + "legendFormat": "g{{group}} {{node_id}}", "range": true, "refId": "A" } ], - "title": "Last Leader Contact", + "title": "FSM Pending by Node", "type": "timeseries" } ], "refresh": "10s", - "schemaVersion": 39, + "schemaVersion": 41, "tags": [ "elastickv", - "raft" + "raft", + "control-plane" ], "templating": { "list": [ { - "current": {}, - "datasource": null, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(elastickv_raft_term{job=\"elastickv\"}, group)", - "hide": 0, "includeAll": true, "label": "Raft Group", "multi": true, "name": "group", "options": [], "query": { + "qryType": 1, "query": "label_values(elastickv_raft_term{job=\"elastickv\"}, group)", - "refId": "StandardVariableQuery" + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\"}, node_id)", + "includeAll": true, + "label": "Node ID", + "multi": true, + "name": "node_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(elastickv_raft_local_state{job=\"elastickv\",group=~\"$group\"}, node_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, + "regex": "", "sort": 1, "type": "query" } ] }, "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Elastickv Raft Status", "uid": "elastickv-raft-status", - "version": 1 + "version": 2 }