gitlord
/
synapse
mirror of https://github.com/matrix-org/synapse.git
1
0
Fork 0
Code Issues 0 Releases 671 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
23258 Commits
842 Branches
288 MiB
 
 
 
 
 
 
Tree: 3df70aa800
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '3df70aa800'
${ noResults }
synapse/contrib/grafana/synapse.json

13474 lines
367 KiB
Raw Blame History 

  1. {

  2.   "__elements": {},

  3.   "__requires": [

  4.     {

  5.       "type": "grafana",

  6.       "id": "grafana",

  7.       "name": "Grafana",

  8.       "version": "9.2.2"

  9.     },

  10.     {

  11.       "type": "panel",

  12.       "id": "graph",

  13.       "name": "Graph (old)",

  14.       "version": ""

  15.     },

  16.     {

  17.       "type": "panel",

  18.       "id": "heatmap",

  19.       "name": "Heatmap",

  20.       "version": ""

  21.     },

  22.     {

  23.       "type": "datasource",

  24.       "id": "prometheus",

  25.       "name": "Prometheus",

  26.       "version": "1.0.0"

  27.     },

  28.     {

  29.       "type": "panel",

  30.       "id": "timeseries",

  31.       "name": "Time series",

  32.       "version": ""

  33.     }

  34.   ],

  35.   "annotations": {

  36.     "list": [

  37.       {

  38.         "builtIn": 1,

  39.         "datasource": {

  40.           "uid": "${DS_PROMETHEUS}"

  41.         },

  42.         "enable": false,

  43.         "hide": true,

  44.         "iconColor": "rgba(0, 211, 255, 1)",

  45.         "limit": 100,

  46.         "name": "Annotations & Alerts",

  47.         "showIn": 0,

  48.         "type": "dashboard"

  49.       },

  50.       {

  51.         "datasource": {

  52.           "type": "prometheus",

  53.           "uid": "${DS_PROMETHEUS}"

  54.         },

  55.         "enable": true,

  56.         "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",

  57.         "iconColor": "purple",

  58.         "name": "deploys",

  59.         "titleFormat": "Deployed {{version}}"

  60.       }

  61.     ]

  62.   },

  63.   "editable": true,

  64.   "fiscalYearStartMonth": 0,

  65.   "graphTooltip": 0,

  66.   "id": null,

  67.   "links": [

  68.     {

  69.       "asDropdown": false,

  70.       "icon": "external link",

  71.       "includeVars": true,

  72.       "keepTime": true,

  73.       "tags": [

  74.         "matrix"

  75.       ],

  76.       "title": "Dashboards",

  77.       "type": "dashboards"

  78.     }

  79.   ],

  80.   "liveNow": false,

  81.   "panels": [

  82.     {

  83.       "collapsed": false,

  84.       "datasource": {

  85.         "type": "prometheus",

  86.         "uid": "${DS_PROMETHEUS}"

  87.       },

  88.       "gridPos": {

  89.         "h": 1,

  90.         "w": 24,

  91.         "x": 0,

  92.         "y": 0

  93.       },

  94.       "id": 73,

  95.       "panels": [],

  96.       "targets": [

  97.         {

  98.           "datasource": {

  99.             "type": "prometheus",

  100.             "uid": "${DS_PROMETHEUS}"

  101.           },

  102.           "refId": "A"

  103.         }

  104.       ],

  105.       "title": "Overview",

  106.       "type": "row"

  107.     },

  108.     {

  109.       "cards": {

  110.         "cardPadding": -1,

  111.         "cardRound": 0

  112.       },

  113.       "color": {

  114.         "cardColor": "#b4ff00",

  115.         "colorScale": "sqrt",

  116.         "colorScheme": "interpolateInferno",

  117.         "exponent": 0.5,

  118.         "mode": "spectrum"

  119.       },

  120.       "dataFormat": "tsbuckets",

  121.       "datasource": {

  122.         "uid": "${DS_PROMETHEUS}"

  123.       },

  124.       "fieldConfig": {

  125.         "defaults": {

  126.           "custom": {

  127.             "hideFrom": {

  128.               "legend": false,

  129.               "tooltip": false,

  130.               "viz": false

  131.             },

  132.             "scaleDistribution": {

  133.               "type": "linear"

  134.             }

  135.           }

  136.         },

  137.         "overrides": []

  138.       },

  139.       "gridPos": {

  140.         "h": 9,

  141.         "w": 12,

  142.         "x": 0,

  143.         "y": 1

  144.       },

  145.       "heatmap": {},

  146.       "hideZeroBuckets": false,

  147.       "highlightCards": true,

  148.       "id": 189,

  149.       "legend": {

  150.         "show": false

  151.       },

  152.       "links": [],

  153.       "options": {

  154.         "calculate": false,

  155.         "calculation": {},

  156.         "cellGap": -1,

  157.         "cellRadius": 0,

  158.         "cellValues": {},

  159.         "color": {

  160.           "exponent": 0.5,

  161.           "fill": "#b4ff00",

  162.           "mode": "scheme",

  163.           "reverse": false,

  164.           "scale": "exponential",

  165.           "scheme": "Inferno",

  166.           "steps": 128

  167.         },

  168.         "exemplars": {

  169.           "color": "rgba(255,0,255,0.7)"

  170.         },

  171.         "filterValues": {

  172.           "le": 1e-9

  173.         },

  174.         "legend": {

  175.           "show": false

  176.         },

  177.         "rowsFrame": {

  178.           "layout": "auto"

  179.         },

  180.         "showValue": "never",

  181.         "tooltip": {

  182.           "show": true,

  183.           "yHistogram": true

  184.         },

  185.         "yAxis": {

  186.           "axisPlacement": "left",

  187.           "reverse": false,

  188.           "unit": "s"

  189.         }

  190.       },

  191.       "pluginVersion": "9.2.2",

  192.       "reverseYBuckets": false,

  193.       "targets": [

  194.         {

  195.           "datasource": {

  196.             "uid": "${DS_PROMETHEUS}"

  197.           },

  198.           "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",

  199.           "format": "heatmap",

  200.           "interval": "",

  201.           "intervalFactor": 1,

  202.           "legendFormat": "{{le}}",

  203.           "refId": "A"

  204.         }

  205.       ],

  206.       "title": "Event Send Time (excluding errors, all workers)",

  207.       "tooltip": {

  208.         "show": true,

  209.         "showHistogram": true

  210.       },

  211.       "type": "heatmap",

  212.       "xAxis": {

  213.         "show": true

  214.       },

  215.       "yAxis": {

  216.         "format": "s",

  217.         "logBase": 2,

  218.         "show": true

  219.       },

  220.       "yBucketBound": "auto"

  221.     },

  222.     {

  223.       "aliasColors": {},

  224.       "bars": false,

  225.       "dashLength": 10,

  226.       "dashes": false,

  227.       "datasource": {

  228.         "uid": "${DS_PROMETHEUS}"

  229.       },

  230.       "description": "",

  231.       "fieldConfig": {

  232.         "defaults": {

  233.           "links": []

  234.         },

  235.         "overrides": []

  236.       },

  237.       "fill": 0,

  238.       "fillGradient": 0,

  239.       "gridPos": {

  240.         "h": 9,

  241.         "w": 12,

  242.         "x": 12,

  243.         "y": 1

  244.       },

  245.       "hiddenSeries": false,

  246.       "id": 152,

  247.       "legend": {

  248.         "avg": false,

  249.         "current": false,

  250.         "max": false,

  251.         "min": false,

  252.         "rightSide": false,

  253.         "show": true,

  254.         "total": false,

  255.         "values": false

  256.       },

  257.       "lines": true,

  258.       "linewidth": 0,

  259.       "links": [],

  260.       "nullPointMode": "connected",

  261.       "options": {

  262.         "alertThreshold": true

  263.       },

  264.       "paceLength": 10,

  265.       "percentage": false,

  266.       "pluginVersion": "9.2.2",

  267.       "pointradius": 5,

  268.       "points": false,

  269.       "renderer": "flot",

  270.       "seriesOverrides": [

  271.         {

  272.           "alias": "Avg",

  273.           "fill": 0,

  274.           "linewidth": 3

  275.         },

  276.         {

  277.           "alias": "99%",

  278.           "color": "#C4162A",

  279.           "fillBelowTo": "90%"

  280.         },

  281.         {

  282.           "alias": "90%",

  283.           "color": "#FF7383",

  284.           "fillBelowTo": "75%"

  285.         },

  286.         {

  287.           "alias": "75%",

  288.           "color": "#FFEE52",

  289.           "fillBelowTo": "50%"

  290.         },

  291.         {

  292.           "alias": "50%",

  293.           "color": "#73BF69",

  294.           "fillBelowTo": "25%"

  295.         },

  296.         {

  297.           "alias": "25%",

  298.           "color": "#1F60C4",

  299.           "fillBelowTo": "5%"

  300.         },

  301.         {

  302.           "alias": "5%",

  303.           "lines": false

  304.         },

  305.         {

  306.           "alias": "Average",

  307.           "color": "rgb(255, 255, 255)",

  308.           "lines": true,

  309.           "linewidth": 3

  310.         },

  311.         {

  312.           "alias": "Events",

  313.           "color": "#B877D9",

  314.           "hideTooltip": true,

  315.           "points": true,

  316.           "yaxis": 2,

  317.           "zindex": -3

  318.         }

  319.       ],

  320.       "spaceLength": 10,

  321.       "stack": false,

  322.       "steppedLine": false,

  323.       "targets": [

  324.         {

  325.           "datasource": {

  326.             "uid": "${DS_PROMETHEUS}"

  327.           },

  328.           "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",

  329.           "format": "time_series",

  330.           "intervalFactor": 1,

  331.           "legendFormat": "99%",

  332.           "refId": "D"

  333.         },

  334.         {

  335.           "datasource": {

  336.             "uid": "${DS_PROMETHEUS}"

  337.           },

  338.           "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",

  339.           "format": "time_series",

  340.           "interval": "",

  341.           "intervalFactor": 1,

  342.           "legendFormat": "90%",

  343.           "refId": "A"

  344.         },

  345.         {

  346.           "datasource": {

  347.             "uid": "${DS_PROMETHEUS}"

  348.           },

  349.           "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",

  350.           "format": "time_series",

  351.           "intervalFactor": 1,

  352.           "legendFormat": "75%",

  353.           "refId": "C"

  354.         },

  355.         {

  356.           "datasource": {

  357.             "uid": "${DS_PROMETHEUS}"

  358.           },

  359.           "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",

  360.           "format": "time_series",

  361.           "intervalFactor": 1,

  362.           "legendFormat": "50%",

  363.           "refId": "B"

  364.         },

  365.         {

  366.           "datasource": {

  367.             "uid": "${DS_PROMETHEUS}"

  368.           },

  369.           "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",

  370.           "legendFormat": "25%",

  371.           "refId": "F"

  372.         },

  373.         {

  374.           "datasource": {

  375.             "uid": "${DS_PROMETHEUS}"

  376.           },

  377.           "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",

  378.           "legendFormat": "5%",

  379.           "refId": "G"

  380.         },

  381.         {

  382.           "datasource": {

  383.             "uid": "${DS_PROMETHEUS}"

  384.           },

  385.           "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",

  386.           "legendFormat": "Average",

  387.           "refId": "H"

  388.         },

  389.         {

  390.           "datasource": {

  391.             "uid": "${DS_PROMETHEUS}"

  392.           },

  393.           "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))",

  394.           "hide": false,

  395.           "instant": false,

  396.           "legendFormat": "Events",

  397.           "refId": "E"

  398.         }

  399.       ],

  400.       "thresholds": [

  401.         {

  402.           "$$hashKey": "object:283",

  403.           "colorMode": "warning",

  404.           "fill": false,

  405.           "line": true,

  406.           "op": "gt",

  407.           "value": 1,

  408.           "yaxis": "left"

  409.         },

  410.         {

  411.           "$$hashKey": "object:284",

  412.           "colorMode": "critical",

  413.           "fill": false,

  414.           "line": true,

  415.           "op": "gt",

  416.           "value": 2,

  417.           "yaxis": "left"

  418.         }

  419.       ],

  420.       "timeRegions": [],

  421.       "title": "Event Send Time Quantiles (excluding errors, all workers)",

  422.       "tooltip": {

  423.         "shared": true,

  424.         "sort": 2,

  425.         "value_type": "individual"

  426.       },

  427.       "type": "graph",

  428.       "xaxis": {

  429.         "mode": "time",

  430.         "show": true,

  431.         "values": []

  432.       },

  433.       "yaxes": [

  434.         {

  435.           "$$hashKey": "object:255",

  436.           "format": "s",

  437.           "label": "",

  438.           "logBase": 1,

  439.           "min": "0",

  440.           "show": true

  441.         },

  442.         {

  443.           "$$hashKey": "object:256",

  444.           "format": "hertz",

  445.           "label": "",

  446.           "logBase": 1,

  447.           "min": "0",

  448.           "show": true

  449.         }

  450.       ],

  451.       "yaxis": {

  452.         "align": false

  453.       }

  454.     },

  455.     {

  456.       "aliasColors": {},

  457.       "bars": false,

  458.       "dashLength": 10,

  459.       "dashes": false,

  460.       "datasource": {

  461.         "uid": "${DS_PROMETHEUS}"

  462.       },

  463.       "fieldConfig": {

  464.         "defaults": {

  465.           "links": []

  466.         },

  467.         "overrides": []

  468.       },

  469.       "fill": 1,

  470.       "fillGradient": 0,

  471.       "gridPos": {

  472.         "h": 9,

  473.         "w": 12,

  474.         "x": 0,

  475.         "y": 10

  476.       },

  477.       "hiddenSeries": false,

  478.       "id": 75,

  479.       "legend": {

  480.         "avg": false,

  481.         "current": false,

  482.         "max": false,

  483.         "min": false,

  484.         "show": true,

  485.         "total": false,

  486.         "values": false

  487.       },

  488.       "lines": true,

  489.       "linewidth": 3,

  490.       "links": [],

  491.       "nullPointMode": "null",

  492.       "options": {

  493.         "alertThreshold": true

  494.       },

  495.       "paceLength": 10,

  496.       "percentage": false,

  497.       "pluginVersion": "9.2.2",

  498.       "pointradius": 5,

  499.       "points": false,

  500.       "renderer": "flot",

  501.       "seriesOverrides": [],

  502.       "spaceLength": 10,

  503.       "stack": false,

  504.       "steppedLine": false,

  505.       "targets": [

  506.         {

  507.           "datasource": {

  508.             "uid": "${DS_PROMETHEUS}"

  509.           },

  510.           "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  511.           "format": "time_series",

  512.           "interval": "",

  513.           "intervalFactor": 1,

  514.           "legendFormat": "{{job}}-{{index}} ",

  515.           "refId": "A"

  516.         }

  517.       ],

  518.       "thresholds": [

  519.         {

  520.           "$$hashKey": "object:566",

  521.           "colorMode": "critical",

  522.           "fill": true,

  523.           "line": true,

  524.           "op": "gt",

  525.           "value": 1,

  526.           "yaxis": "left"

  527.         }

  528.       ],

  529.       "timeRegions": [],

  530.       "title": "CPU usage",

  531.       "tooltip": {

  532.         "shared": false,

  533.         "sort": 0,

  534.         "value_type": "individual"

  535.       },

  536.       "type": "graph",

  537.       "xaxis": {

  538.         "mode": "time",

  539.         "show": true,

  540.         "values": []

  541.       },

  542.       "yaxes": [

  543.         {

  544.           "$$hashKey": "object:538",

  545.           "format": "percentunit",

  546.           "logBase": 1,

  547.           "max": "1.5",

  548.           "min": "0",

  549.           "show": true

  550.         },

  551.         {

  552.           "$$hashKey": "object:539",

  553.           "format": "short",

  554.           "logBase": 1,

  555.           "show": true

  556.         }

  557.       ],

  558.       "yaxis": {

  559.         "align": false

  560.       }

  561.     },

  562.     {

  563.       "aliasColors": {},

  564.       "bars": false,

  565.       "dashLength": 10,

  566.       "dashes": false,

  567.       "datasource": {

  568.         "uid": "${DS_PROMETHEUS}"

  569.       },

  570.       "editable": true,

  571.       "error": false,

  572.       "fieldConfig": {

  573.         "defaults": {

  574.           "links": []

  575.         },

  576.         "overrides": []

  577.       },

  578.       "fill": 1,

  579.       "fillGradient": 0,

  580.       "grid": {},

  581.       "gridPos": {

  582.         "h": 9,

  583.         "w": 12,

  584.         "x": 12,

  585.         "y": 10

  586.       },

  587.       "hiddenSeries": false,

  588.       "id": 198,

  589.       "legend": {

  590.         "avg": false,

  591.         "current": false,

  592.         "max": false,

  593.         "min": false,

  594.         "show": true,

  595.         "total": false,

  596.         "values": false

  597.       },

  598.       "lines": true,

  599.       "linewidth": 3,

  600.       "links": [],

  601.       "nullPointMode": "null",

  602.       "options": {

  603.         "alertThreshold": true

  604.       },

  605.       "paceLength": 10,

  606.       "percentage": false,

  607.       "pluginVersion": "9.2.2",

  608.       "pointradius": 5,

  609.       "points": false,

  610.       "renderer": "flot",

  611.       "seriesOverrides": [],

  612.       "spaceLength": 10,

  613.       "stack": false,

  614.       "steppedLine": false,

  615.       "targets": [

  616.         {

  617.           "datasource": {

  618.             "uid": "${DS_PROMETHEUS}"

  619.           },

  620.           "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  621.           "format": "time_series",

  622.           "interval": "",

  623.           "intervalFactor": 2,

  624.           "legendFormat": "{{job}} {{index}}",

  625.           "refId": "A",

  626.           "step": 20,

  627.           "target": ""

  628.         },

  629.         {

  630.           "datasource": {

  631.             "uid": "${DS_PROMETHEUS}"

  632.           },

  633.           "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",

  634.           "hide": true,

  635.           "interval": "",

  636.           "legendFormat": "total",

  637.           "refId": "B"

  638.         }

  639.       ],

  640.       "thresholds": [],

  641.       "timeRegions": [],

  642.       "title": "Memory",

  643.       "tooltip": {

  644.         "shared": false,

  645.         "sort": 0,

  646.         "value_type": "cumulative"

  647.       },

  648.       "transformations": [],

  649.       "type": "graph",

  650.       "xaxis": {

  651.         "mode": "time",

  652.         "show": true,

  653.         "values": []

  654.       },

  655.       "yaxes": [

  656.         {

  657.           "$$hashKey": "object:1560",

  658.           "format": "bytes",

  659.           "logBase": 1,

  660.           "min": "0",

  661.           "show": true

  662.         },

  663.         {

  664.           "$$hashKey": "object:1561",

  665.           "format": "short",

  666.           "logBase": 1,

  667.           "show": true

  668.         }

  669.       ],

  670.       "yaxis": {

  671.         "align": false

  672.       }

  673.     },

  674.     {

  675.       "datasource": {

  676.         "type": "prometheus",

  677.         "uid": "${DS_PROMETHEUS}"

  678.       },

  679.       "fieldConfig": {

  680.         "defaults": {

  681.           "color": {

  682.             "mode": "palette-classic"

  683.           },

  684.           "custom": {

  685.             "axisCenteredZero": false,

  686.             "axisColorMode": "text",

  687.             "axisLabel": "",

  688.             "axisPlacement": "auto",

  689.             "axisSoftMax": 1,

  690.             "barAlignment": 0,

  691.             "drawStyle": "line",

  692.             "fillOpacity": 0,

  693.             "gradientMode": "none",

  694.             "hideFrom": {

  695.               "legend": false,

  696.               "tooltip": false,

  697.               "viz": false

  698.             },

  699.             "lineInterpolation": "linear",

  700.             "lineWidth": 10,

  701.             "pointSize": 5,

  702.             "scaleDistribution": {

  703.               "type": "linear"

  704.             },

  705.             "showPoints": "never",

  706.             "spanNulls": false,

  707.             "stacking": {

  708.               "group": "A",

  709.               "mode": "none"

  710.             },

  711.             "thresholdsStyle": {

  712.               "mode": "off"

  713.             }

  714.           },

  715.           "mappings": [],

  716.           "thresholds": {

  717.             "mode": "absolute",

  718.             "steps": [

  719.               {

  720.                 "color": "green",

  721.                 "value": null

  722.               }

  723.             ]

  724.           }

  725.         },

  726.         "overrides": []

  727.       },

  728.       "gridPos": {

  729.         "h": 7,

  730.         "w": 12,

  731.         "x": 0,

  732.         "y": 19

  733.       },

  734.       "id": 245,

  735.       "options": {

  736.         "legend": {

  737.           "calcs": [],

  738.           "displayMode": "list",

  739.           "placement": "bottom",

  740.           "showLegend": true

  741.         },

  742.         "tooltip": {

  743.           "mode": "single",

  744.           "sort": "none"

  745.         }

  746.       },

  747.       "targets": [

  748.         {

  749.           "datasource": {

  750.             "type": "prometheus",

  751.             "uid": "${DS_PROMETHEUS}"

  752.           },

  753.           "editorMode": "code",

  754.           "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",

  755.           "legendFormat": "version {{version}}",

  756.           "range": true,

  757.           "refId": "deployed_synapse_versions"

  758.         }

  759.       ],

  760.       "title": "Deployed Synapse versions over time",

  761.       "type": "timeseries"

  762.     },

  763.     {

  764.       "aliasColors": {},

  765.       "bars": false,

  766.       "dashLength": 10,

  767.       "dashes": false,

  768.       "datasource": {

  769.         "uid": "${DS_PROMETHEUS}"

  770.       },

  771.       "fieldConfig": {

  772.         "defaults": {

  773.           "links": []

  774.         },

  775.         "overrides": []

  776.       },

  777.       "fill": 1,

  778.       "fillGradient": 0,

  779.       "gridPos": {

  780.         "h": 7,

  781.         "w": 12,

  782.         "x": 12,

  783.         "y": 19

  784.       },

  785.       "hiddenSeries": false,

  786.       "id": 37,

  787.       "legend": {

  788.         "avg": false,

  789.         "current": false,

  790.         "max": false,

  791.         "min": false,

  792.         "show": true,

  793.         "total": false,

  794.         "values": false

  795.       },

  796.       "lines": true,

  797.       "linewidth": 1,

  798.       "links": [],

  799.       "nullPointMode": "null",

  800.       "options": {

  801.         "alertThreshold": true

  802.       },

  803.       "paceLength": 10,

  804.       "percentage": false,

  805.       "pluginVersion": "9.2.2",

  806.       "pointradius": 5,

  807.       "points": false,

  808.       "renderer": "flot",

  809.       "seriesOverrides": [

  810.         {

  811.           "$$hashKey": "object:639",

  812.           "alias": "/max$/",

  813.           "color": "#890F02",

  814.           "fill": 0,

  815.           "legend": false

  816.         }

  817.       ],

  818.       "spaceLength": 10,

  819.       "stack": false,

  820.       "steppedLine": false,

  821.       "targets": [

  822.         {

  823.           "datasource": {

  824.             "uid": "${DS_PROMETHEUS}"

  825.           },

  826.           "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  827.           "format": "time_series",

  828.           "hide": false,

  829.           "interval": "",

  830.           "intervalFactor": 2,

  831.           "legendFormat": "{{job}}-{{index}}",

  832.           "refId": "A",

  833.           "step": 20

  834.         },

  835.         {

  836.           "datasource": {

  837.             "uid": "${DS_PROMETHEUS}"

  838.           },

  839.           "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  840.           "format": "time_series",

  841.           "hide": true,

  842.           "interval": "",

  843.           "intervalFactor": 2,

  844.           "legendFormat": "{{job}}-{{index}} max",

  845.           "refId": "B",

  846.           "step": 20

  847.         }

  848.       ],

  849.       "thresholds": [],

  850.       "timeRegions": [],

  851.       "title": "Open FDs",

  852.       "tooltip": {

  853.         "shared": false,

  854.         "sort": 0,

  855.         "value_type": "individual"

  856.       },

  857.       "type": "graph",

  858.       "xaxis": {

  859.         "mode": "time",

  860.         "show": true,

  861.         "values": []

  862.       },

  863.       "yaxes": [

  864.         {

  865.           "$$hashKey": "object:650",

  866.           "format": "none",

  867.           "label": "",

  868.           "logBase": 1,

  869.           "show": true

  870.         },

  871.         {

  872.           "$$hashKey": "object:651",

  873.           "format": "short",

  874.           "logBase": 1,

  875.           "show": true

  876.         }

  877.       ],

  878.       "yaxis": {

  879.         "align": false

  880.       }

  881.     },

  882.     {

  883.       "collapsed": true,

  884.       "datasource": {

  885.         "type": "prometheus",

  886.         "uid": "${DS_PROMETHEUS}"

  887.       },

  888.       "gridPos": {

  889.         "h": 1,

  890.         "w": 24,

  891.         "x": 0,

  892.         "y": 26

  893.       },

  894.       "id": 54,

  895.       "panels": [

  896.         {

  897.           "aliasColors": {},

  898.           "bars": false,

  899.           "dashLength": 10,

  900.           "dashes": false,

  901.           "datasource": {

  902.             "type": "prometheus",

  903.             "uid": "${DS_PROMETHEUS}"

  904.           },

  905.           "editable": true,

  906.           "error": false,

  907.           "fieldConfig": {

  908.             "defaults": {

  909.               "links": []

  910.             },

  911.             "overrides": []

  912.           },

  913.           "fill": 1,

  914.           "fillGradient": 0,

  915.           "grid": {},

  916.           "gridPos": {

  917.             "h": 7,

  918.             "w": 12,

  919.             "x": 0,

  920.             "y": 27

  921.           },

  922.           "hiddenSeries": false,

  923.           "id": 5,

  924.           "legend": {

  925.             "alignAsTable": false,

  926.             "avg": false,

  927.             "current": false,

  928.             "hideEmpty": false,

  929.             "hideZero": false,

  930.             "max": false,

  931.             "min": false,

  932.             "rightSide": false,

  933.             "show": true,

  934.             "total": false,

  935.             "values": false

  936.           },

  937.           "lines": true,

  938.           "linewidth": 3,

  939.           "links": [],

  940.           "nullPointMode": "null",

  941.           "options": {

  942.             "alertThreshold": true

  943.           },

  944.           "paceLength": 10,

  945.           "percentage": false,

  946.           "pluginVersion": "9.2.2",

  947.           "pointradius": 5,

  948.           "points": false,

  949.           "renderer": "flot",

  950.           "seriesOverrides": [

  951.             {

  952.               "$$hashKey": "object:1240",

  953.               "alias": "/user/"

  954.             },

  955.             {

  956.               "$$hashKey": "object:1241",

  957.               "alias": "/system/"

  958.             }

  959.           ],

  960.           "spaceLength": 10,

  961.           "stack": false,

  962.           "steppedLine": false,

  963.           "targets": [

  964.             {

  965.               "datasource": {

  966.                 "uid": "${DS_PROMETHEUS}"

  967.               },

  968.               "editorMode": "code",

  969.               "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  970.               "format": "time_series",

  971.               "intervalFactor": 1,

  972.               "legendFormat": "{{job}}-{{index}} system ",

  973.               "metric": "",

  974.               "range": true,

  975.               "refId": "B",

  976.               "step": 20

  977.             },

  978.             {

  979.               "datasource": {

  980.                 "uid": "${DS_PROMETHEUS}"

  981.               },

  982.               "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  983.               "format": "time_series",

  984.               "hide": false,

  985.               "interval": "",

  986.               "intervalFactor": 1,

  987.               "legendFormat": "{{job}}-{{index}} user",

  988.               "refId": "A",

  989.               "step": 20

  990.             }

  991.           ],

  992.           "thresholds": [

  993.             {

  994.               "$$hashKey": "object:1278",

  995.               "colorMode": "custom",

  996.               "fillColor": "rgba(255, 255, 255, 1)",

  997.               "line": true,

  998.               "lineColor": "rgba(216, 200, 27, 0.27)",

  999.               "op": "gt",

  1000.               "value": 0.5,

  1001.               "yaxis": "left"

  1002.             },

  1003.             {

  1004.               "$$hashKey": "object:1279",

  1005.               "colorMode": "custom",

  1006.               "fillColor": "rgba(255, 255, 255, 1)",

  1007.               "line": true,

  1008.               "lineColor": "rgb(87, 6, 16)",

  1009.               "op": "gt",

  1010.               "value": 0.8,

  1011.               "yaxis": "left"

  1012.             },

  1013.             {

  1014.               "$$hashKey": "object:1498",

  1015.               "colorMode": "critical",

  1016.               "fill": true,

  1017.               "line": true,

  1018.               "op": "gt",

  1019.               "value": 1,

  1020.               "yaxis": "left"

  1021.             }

  1022.           ],

  1023.           "timeRegions": [],

  1024.           "title": "CPU",

  1025.           "tooltip": {

  1026.             "shared": false,

  1027.             "sort": 0,

  1028.             "value_type": "individual"

  1029.           },

  1030.           "type": "graph",

  1031.           "xaxis": {

  1032.             "mode": "time",

  1033.             "show": true,

  1034.             "values": []

  1035.           },

  1036.           "yaxes": [

  1037.             {

  1038.               "$$hashKey": "object:1250",

  1039.               "format": "percentunit",

  1040.               "label": "",

  1041.               "logBase": 1,

  1042.               "max": "1.2",

  1043.               "min": 0,

  1044.               "show": true

  1045.             },

  1046.             {

  1047.               "$$hashKey": "object:1251",

  1048.               "format": "short",

  1049.               "logBase": 1,

  1050.               "show": true

  1051.             }

  1052.           ],

  1053.           "yaxis": {

  1054.             "align": false

  1055.           }

  1056.         },

  1057.         {

  1058.           "datasource": {

  1059.             "type": "prometheus",

  1060.             "uid": "${DS_PROMETHEUS}"

  1061.           },

  1062.           "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",

  1063.           "fieldConfig": {

  1064.             "defaults": {

  1065.               "color": {

  1066.                 "mode": "palette-classic"

  1067.               },

  1068.               "custom": {

  1069.                 "axisCenteredZero": false,

  1070.                 "axisColorMode": "text",

  1071.                 "axisLabel": "",

  1072.                 "axisPlacement": "auto",

  1073.                 "barAlignment": 0,

  1074.                 "drawStyle": "line",

  1075.                 "fillOpacity": 10,

  1076.                 "gradientMode": "none",

  1077.                 "hideFrom": {

  1078.                   "legend": false,

  1079.                   "tooltip": false,

  1080.                   "viz": false

  1081.                 },

  1082.                 "lineInterpolation": "linear",

  1083.                 "lineWidth": 1,

  1084.                 "pointSize": 5,

  1085.                 "scaleDistribution": {

  1086.                   "type": "linear"

  1087.                 },

  1088.                 "showPoints": "never",

  1089.                 "spanNulls": true,

  1090.                 "stacking": {

  1091.                   "group": "A",

  1092.                   "mode": "none"

  1093.                 },

  1094.                 "thresholdsStyle": {

  1095.                   "mode": "off"

  1096.                 }

  1097.               },

  1098.               "links": [],

  1099.               "mappings": [],

  1100.               "thresholds": {

  1101.                 "mode": "absolute",

  1102.                 "steps": [

  1103.                   {

  1104.                     "color": "green"

  1105.                   },

  1106.                   {

  1107.                     "color": "red",

  1108.                     "value": 80

  1109.                   }

  1110.                 ]

  1111.               },

  1112.               "unit": "s"

  1113.             },

  1114.             "overrides": []

  1115.           },

  1116.           "gridPos": {

  1117.             "h": 7,

  1118.             "w": 12,

  1119.             "x": 12,

  1120.             "y": 27

  1121.           },

  1122.           "id": 105,

  1123.           "interval": "",

  1124.           "links": [],

  1125.           "options": {

  1126.             "legend": {

  1127.               "calcs": [],

  1128.               "displayMode": "list",

  1129.               "placement": "bottom",

  1130.               "showLegend": true

  1131.             },

  1132.             "tooltip": {

  1133.               "mode": "single",

  1134.               "sort": "none"

  1135.             }

  1136.           },

  1137.           "pluginVersion": "8.3.2",

  1138.           "targets": [

  1139.             {

  1140.               "datasource": {

  1141.                 "type": "prometheus",

  1142.                 "uid": "${DS_PROMETHEUS}"

  1143.               },

  1144.               "exemplar": true,

  1145.               "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",

  1146.               "hide": false,

  1147.               "interval": "",

  1148.               "legendFormat": "{{job}}-{{index}} 99.9%",

  1149.               "refId": "E"

  1150.             },

  1151.             {

  1152.               "datasource": {

  1153.                 "type": "prometheus",

  1154.                 "uid": "${DS_PROMETHEUS}"

  1155.               },

  1156.               "exemplar": true,

  1157.               "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",

  1158.               "format": "time_series",

  1159.               "interval": "",

  1160.               "intervalFactor": 2,

  1161.               "legendFormat": "{{job}}-{{index}} 99%",

  1162.               "refId": "A",

  1163.               "step": 20

  1164.             },

  1165.             {

  1166.               "datasource": {

  1167.                 "type": "prometheus",

  1168.                 "uid": "${DS_PROMETHEUS}"

  1169.               },

  1170.               "exemplar": true,

  1171.               "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",

  1172.               "format": "time_series",

  1173.               "interval": "",

  1174.               "intervalFactor": 1,

  1175.               "legendFormat": "{{job}}-{{index}} 95%",

  1176.               "refId": "B"

  1177.             },

  1178.             {

  1179.               "datasource": {

  1180.                 "type": "prometheus",

  1181.                 "uid": "${DS_PROMETHEUS}"

  1182.               },

  1183.               "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",

  1184.               "format": "time_series",

  1185.               "intervalFactor": 1,

  1186.               "legendFormat": "{{job}}-{{index}} 90%",

  1187.               "refId": "C"

  1188.             },

  1189.             {

  1190.               "datasource": {

  1191.                 "type": "prometheus",

  1192.                 "uid": "${DS_PROMETHEUS}"

  1193.               },

  1194.               "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",

  1195.               "format": "time_series",

  1196.               "intervalFactor": 1,

  1197.               "legendFormat": "{{job}}-{{index}} mean",

  1198.               "refId": "D"

  1199.             }

  1200.           ],

  1201.           "title": "Reactor tick quantiles",

  1202.           "type": "timeseries"

  1203.         },

  1204.         {

  1205.           "aliasColors": {},

  1206.           "bars": false,

  1207.           "dashLength": 10,

  1208.           "dashes": false,

  1209.           "datasource": {

  1210.             "uid": "${DS_PROMETHEUS}"

  1211.           },

  1212.           "editable": true,

  1213.           "error": false,

  1214.           "fieldConfig": {

  1215.             "defaults": {

  1216.               "links": []

  1217.             },

  1218.             "overrides": []

  1219.           },

  1220.           "fill": 0,

  1221.           "fillGradient": 0,

  1222.           "grid": {},

  1223.           "gridPos": {

  1224.             "h": 7,

  1225.             "w": 12,

  1226.             "x": 0,

  1227.             "y": 34

  1228.           },

  1229.           "hiddenSeries": false,

  1230.           "id": 34,

  1231.           "legend": {

  1232.             "avg": false,

  1233.             "current": false,

  1234.             "max": false,

  1235.             "min": false,

  1236.             "show": true,

  1237.             "total": false,

  1238.             "values": false

  1239.           },

  1240.           "lines": true,

  1241.           "linewidth": 1,

  1242.           "links": [],

  1243.           "nullPointMode": "null",

  1244.           "options": {

  1245.             "alertThreshold": true

  1246.           },

  1247.           "paceLength": 10,

  1248.           "percentage": false,

  1249.           "pluginVersion": "9.2.2",

  1250.           "pointradius": 5,

  1251.           "points": false,

  1252.           "renderer": "flot",

  1253.           "seriesOverrides": [],

  1254.           "spaceLength": 10,

  1255.           "stack": false,

  1256.           "steppedLine": false,

  1257.           "targets": [

  1258.             {

  1259.               "datasource": {

  1260.                 "uid": "${DS_PROMETHEUS}"

  1261.               },

  1262.               "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  1263.               "format": "time_series",

  1264.               "interval": "",

  1265.               "intervalFactor": 2,

  1266.               "legendFormat": "{{job}} {{index}}",

  1267.               "refId": "A",

  1268.               "step": 20,

  1269.               "target": ""

  1270.             },

  1271.             {

  1272.               "datasource": {

  1273.                 "uid": "${DS_PROMETHEUS}"

  1274.               },

  1275.               "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",

  1276.               "interval": "",

  1277.               "legendFormat": "total",

  1278.               "refId": "B"

  1279.             }

  1280.           ],

  1281.           "thresholds": [],

  1282.           "timeRegions": [],

  1283.           "title": "Memory",

  1284.           "tooltip": {

  1285.             "shared": false,

  1286.             "sort": 0,

  1287.             "value_type": "cumulative"

  1288.           },

  1289.           "transformations": [],

  1290.           "type": "graph",

  1291.           "xaxis": {

  1292.             "mode": "time",

  1293.             "show": true,

  1294.             "values": []

  1295.           },

  1296.           "yaxes": [

  1297.             {

  1298.               "format": "bytes",

  1299.               "logBase": 1,

  1300.               "min": "0",

  1301.               "show": true

  1302.             },

  1303.             {

  1304.               "format": "short",

  1305.               "logBase": 1,

  1306.               "show": true

  1307.             }

  1308.           ],

  1309.           "yaxis": {

  1310.             "align": false

  1311.           }

  1312.         },

  1313.         {

  1314.           "aliasColors": {},

  1315.           "bars": false,

  1316.           "dashLength": 10,

  1317.           "dashes": false,

  1318.           "datasource": {

  1319.             "uid": "${DS_PROMETHEUS}"

  1320.           },

  1321.           "fieldConfig": {

  1322.             "defaults": {

  1323.               "links": []

  1324.             },

  1325.             "overrides": []

  1326.           },

  1327.           "fill": 1,

  1328.           "fillGradient": 0,

  1329.           "gridPos": {

  1330.             "h": 7,

  1331.             "w": 12,

  1332.             "x": 12,

  1333.             "y": 34

  1334.           },

  1335.           "hiddenSeries": false,

  1336.           "id": 49,

  1337.           "legend": {

  1338.             "avg": false,

  1339.             "current": false,

  1340.             "max": false,

  1341.             "min": false,

  1342.             "show": true,

  1343.             "total": false,

  1344.             "values": false

  1345.           },

  1346.           "lines": true,

  1347.           "linewidth": 1,

  1348.           "links": [],

  1349.           "nullPointMode": "null",

  1350.           "options": {

  1351.             "alertThreshold": true

  1352.           },

  1353.           "paceLength": 10,

  1354.           "percentage": false,

  1355.           "pluginVersion": "9.2.2",

  1356.           "pointradius": 5,

  1357.           "points": false,

  1358.           "renderer": "flot",

  1359.           "seriesOverrides": [

  1360.             {

  1361.               "alias": "/^up/",

  1362.               "legend": false,

  1363.               "yaxis": 2

  1364.             }

  1365.           ],

  1366.           "spaceLength": 10,

  1367.           "stack": false,

  1368.           "steppedLine": false,

  1369.           "targets": [

  1370.             {

  1371.               "datasource": {

  1372.                 "uid": "${DS_PROMETHEUS}"

  1373.               },

  1374.               "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  1375.               "format": "time_series",

  1376.               "interval": "",

  1377.               "intervalFactor": 2,

  1378.               "legendFormat": "{{job}}-{{index}}",

  1379.               "refId": "A",

  1380.               "step": 20

  1381.             }

  1382.           ],

  1383.           "thresholds": [],

  1384.           "timeRegions": [],

  1385.           "title": "Prometheus scrape time",

  1386.           "tooltip": {

  1387.             "shared": false,

  1388.             "sort": 0,

  1389.             "value_type": "individual"

  1390.           },

  1391.           "type": "graph",

  1392.           "xaxis": {

  1393.             "mode": "time",

  1394.             "show": true,

  1395.             "values": []

  1396.           },

  1397.           "yaxes": [

  1398.             {

  1399.               "format": "s",

  1400.               "logBase": 1,

  1401.               "min": "0",

  1402.               "show": true

  1403.             },

  1404.             {

  1405.               "decimals": 0,

  1406.               "format": "none",

  1407.               "label": "",

  1408.               "logBase": 1,

  1409.               "max": "0",

  1410.               "min": "-1",

  1411.               "show": false

  1412.             }

  1413.           ],

  1414.           "yaxis": {

  1415.             "align": false

  1416.           }

  1417.         },

  1418.         {

  1419.           "aliasColors": {},

  1420.           "bars": false,

  1421.           "dashLength": 10,

  1422.           "dashes": false,

  1423.           "datasource": {

  1424.             "type": "prometheus",

  1425.             "uid": "${DS_PROMETHEUS}"

  1426.           },

  1427.           "fieldConfig": {

  1428.             "defaults": {

  1429.               "links": []

  1430.             },

  1431.             "overrides": []

  1432.           },

  1433.           "fill": 0,

  1434.           "fillGradient": 0,

  1435.           "gridPos": {

  1436.             "h": 7,

  1437.             "w": 12,

  1438.             "x": 0,

  1439.             "y": 41

  1440.           },

  1441.           "hiddenSeries": false,

  1442.           "id": 53,

  1443.           "legend": {

  1444.             "avg": false,

  1445.             "current": false,

  1446.             "max": false,

  1447.             "min": false,

  1448.             "show": true,

  1449.             "total": false,

  1450.             "values": false

  1451.           },

  1452.           "lines": true,

  1453.           "linewidth": 1,

  1454.           "links": [],

  1455.           "nullPointMode": "null",

  1456.           "options": {

  1457.             "alertThreshold": true

  1458.           },

  1459.           "paceLength": 10,

  1460.           "percentage": false,

  1461.           "pluginVersion": "9.2.2",

  1462.           "pointradius": 5,

  1463.           "points": false,

  1464.           "renderer": "flot",

  1465.           "seriesOverrides": [

  1466.             {

  1467.               "$$hashKey": "object:116",

  1468.               "alias": "/^version .*/",

  1469.               "lines": true,

  1470.               "linewidth": 6,

  1471.               "points": false

  1472.             }

  1473.           ],

  1474.           "spaceLength": 10,

  1475.           "stack": false,

  1476.           "steppedLine": false,

  1477.           "targets": [

  1478.             {

  1479.               "datasource": {

  1480.                 "uid": "${DS_PROMETHEUS}"

  1481.               },

  1482.               "editorMode": "code",

  1483.               "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  1484.               "format": "time_series",

  1485.               "intervalFactor": 2,

  1486.               "legendFormat": "{{job}}-{{index}}",

  1487.               "range": true,

  1488.               "refId": "A"

  1489.             },

  1490.             {

  1491.               "datasource": {

  1492.                 "type": "prometheus",

  1493.                 "uid": "${DS_PROMETHEUS}"

  1494.               },

  1495.               "editorMode": "code",

  1496.               "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",

  1497.               "hide": false,

  1498.               "legendFormat": "version {{version}}",

  1499.               "range": true,

  1500.               "refId": "deployed_synapse_versions"

  1501.             }

  1502.           ],

  1503.           "thresholds": [],

  1504.           "timeRegions": [],

  1505.           "title": "Up",

  1506.           "tooltip": {

  1507.             "shared": false,

  1508.             "sort": 0,

  1509.             "value_type": "individual"

  1510.           },

  1511.           "type": "graph",

  1512.           "xaxis": {

  1513.             "mode": "time",

  1514.             "show": true,

  1515.             "values": []

  1516.           },

  1517.           "yaxes": [

  1518.             {

  1519.               "format": "short",

  1520.               "logBase": 1,

  1521.               "show": true

  1522.             },

  1523.             {

  1524.               "format": "short",

  1525.               "logBase": 1,

  1526.               "show": true

  1527.             }

  1528.           ],

  1529.           "yaxis": {

  1530.             "align": false

  1531.           }

  1532.         },

  1533.         {

  1534.           "aliasColors": {},

  1535.           "bars": false,

  1536.           "dashLength": 10,

  1537.           "dashes": false,

  1538.           "datasource": {

  1539.             "uid": "${DS_PROMETHEUS}"

  1540.           },

  1541.           "fieldConfig": {

  1542.             "defaults": {

  1543.               "links": []

  1544.             },

  1545.             "overrides": []

  1546.           },

  1547.           "fill": 1,

  1548.           "fillGradient": 0,

  1549.           "gridPos": {

  1550.             "h": 7,

  1551.             "w": 12,

  1552.             "x": 12,

  1553.             "y": 41

  1554.           },

  1555.           "hiddenSeries": false,

  1556.           "id": 120,

  1557.           "legend": {

  1558.             "avg": false,

  1559.             "current": false,

  1560.             "max": false,

  1561.             "min": false,

  1562.             "show": true,

  1563.             "total": false,

  1564.             "values": false

  1565.           },

  1566.           "lines": true,

  1567.           "linewidth": 1,

  1568.           "links": [],

  1569.           "nullPointMode": "null as zero",

  1570.           "options": {

  1571.             "alertThreshold": true

  1572.           },

  1573.           "percentage": false,

  1574.           "pluginVersion": "9.2.2",

  1575.           "pointradius": 2,

  1576.           "points": false,

  1577.           "renderer": "flot",

  1578.           "seriesOverrides": [],

  1579.           "spaceLength": 10,

  1580.           "stack": true,

  1581.           "steppedLine": false,

  1582.           "targets": [

  1583.             {

  1584.               "datasource": {

  1585.                 "uid": "${DS_PROMETHEUS}"

  1586.               },

  1587.               "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  1588.               "format": "time_series",

  1589.               "hide": false,

  1590.               "instant": false,

  1591.               "intervalFactor": 1,

  1592.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",

  1593.               "refId": "A"

  1594.             },

  1595.             {

  1596.               "datasource": {

  1597.                 "uid": "${DS_PROMETHEUS}"

  1598.               },

  1599.               "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  1600.               "format": "time_series",

  1601.               "hide": false,

  1602.               "instant": false,

  1603.               "interval": "",

  1604.               "intervalFactor": 1,

  1605.               "legendFormat": "{{job}}-{{index}} {{name}}",

  1606.               "refId": "B"

  1607.             }

  1608.           ],

  1609.           "thresholds": [

  1610.             {

  1611.               "colorMode": "critical",

  1612.               "fill": true,

  1613.               "line": true,

  1614.               "op": "gt",

  1615.               "value": 1,

  1616.               "yaxis": "left"

  1617.             }

  1618.           ],

  1619.           "timeRegions": [],

  1620.           "title": "Stacked CPU usage",

  1621.           "tooltip": {

  1622.             "shared": false,

  1623.             "sort": 0,

  1624.             "value_type": "individual"

  1625.           },

  1626.           "type": "graph",

  1627.           "xaxis": {

  1628.             "mode": "time",

  1629.             "show": true,

  1630.             "values": []

  1631.           },

  1632.           "yaxes": [

  1633.             {

  1634.               "$$hashKey": "object:572",

  1635.               "format": "percentunit",

  1636.               "logBase": 1,

  1637.               "show": true

  1638.             },

  1639.             {

  1640.               "$$hashKey": "object:573",

  1641.               "format": "short",

  1642.               "logBase": 1,

  1643.               "show": true

  1644.             }

  1645.           ],

  1646.           "yaxis": {

  1647.             "align": false

  1648.           }

  1649.         },

  1650.         {

  1651.           "aliasColors": {},

  1652.           "bars": false,

  1653.           "dashLength": 10,

  1654.           "dashes": false,

  1655.           "datasource": {

  1656.             "type": "prometheus",

  1657.             "uid": "${DS_PROMETHEUS}"

  1658.           },

  1659.           "fieldConfig": {

  1660.             "defaults": {

  1661.               "links": []

  1662.             },

  1663.             "overrides": []

  1664.           },

  1665.           "fill": 1,

  1666.           "fillGradient": 0,

  1667.           "gridPos": {

  1668.             "h": 7,

  1669.             "w": 12,

  1670.             "x": 0,

  1671.             "y": 48

  1672.           },

  1673.           "hiddenSeries": false,

  1674.           "id": 136,

  1675.           "interval": "",

  1676.           "legend": {

  1677.             "avg": false,

  1678.             "current": false,

  1679.             "max": false,

  1680.             "min": false,

  1681.             "show": true,

  1682.             "total": false,

  1683.             "values": false

  1684.           },

  1685.           "lines": true,

  1686.           "linewidth": 1,

  1687.           "nullPointMode": "null",

  1688.           "options": {

  1689.             "alertThreshold": true

  1690.           },

  1691.           "percentage": false,

  1692.           "pluginVersion": "9.2.2",

  1693.           "pointradius": 2,

  1694.           "points": false,

  1695.           "renderer": "flot",

  1696.           "seriesOverrides": [],

  1697.           "spaceLength": 10,

  1698.           "stack": false,

  1699.           "steppedLine": false,

  1700.           "targets": [

  1701.             {

  1702.               "datasource": {

  1703.                 "uid": "${DS_PROMETHEUS}"

  1704.               },

  1705.               "editorMode": "code",

  1706.               "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  1707.               "legendFormat": "{{job}}-{{index}} {{method}}",

  1708.               "range": true,

  1709.               "refId": "A"

  1710.             },

  1711.             {

  1712.               "datasource": {

  1713.                 "uid": "${DS_PROMETHEUS}"

  1714.               },

  1715.               "editorMode": "code",

  1716.               "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  1717.               "legendFormat": "{{job}}-{{index}} {{method}} (federation)",

  1718.               "range": true,

  1719.               "refId": "B"

  1720.             }

  1721.           ],

  1722.           "thresholds": [],

  1723.           "timeRegions": [],

  1724.           "title": "Outgoing HTTP request rate",

  1725.           "tooltip": {

  1726.             "shared": false,

  1727.             "sort": 0,

  1728.             "value_type": "individual"

  1729.           },

  1730.           "type": "graph",

  1731.           "xaxis": {

  1732.             "mode": "time",

  1733.             "show": true,

  1734.             "values": []

  1735.           },

  1736.           "yaxes": [

  1737.             {

  1738.               "$$hashKey": "object:123",

  1739.               "format": "reqps",

  1740.               "logBase": 1,

  1741.               "show": true

  1742.             },

  1743.             {

  1744.               "$$hashKey": "object:124",

  1745.               "format": "short",

  1746.               "logBase": 1,

  1747.               "show": true

  1748.             }

  1749.           ],

  1750.           "yaxis": {

  1751.             "align": false

  1752.           }

  1753.         },

  1754.         {

  1755.           "datasource": {

  1756.             "type": "prometheus",

  1757.             "uid": "${DS_PROMETHEUS}"

  1758.           },

  1759.           "fieldConfig": {

  1760.             "defaults": {

  1761.               "color": {

  1762.                 "mode": "palette-classic"

  1763.               },

  1764.               "custom": {

  1765.                 "axisCenteredZero": false,

  1766.                 "axisColorMode": "text",

  1767.                 "axisLabel": "active threads",

  1768.                 "axisPlacement": "auto",

  1769.                 "barAlignment": 0,

  1770.                 "drawStyle": "line",

  1771.                 "fillOpacity": 0,

  1772.                 "gradientMode": "none",

  1773.                 "hideFrom": {

  1774.                   "legend": false,

  1775.                   "tooltip": false,

  1776.                   "viz": false

  1777.                 },

  1778.                 "lineInterpolation": "linear",

  1779.                 "lineWidth": 1,

  1780.                 "pointSize": 5,

  1781.                 "scaleDistribution": {

  1782.                   "type": "linear"

  1783.                 },

  1784.                 "showPoints": "auto",

  1785.                 "spanNulls": false,

  1786.                 "stacking": {

  1787.                   "group": "A",

  1788.                   "mode": "none"

  1789.                 },

  1790.                 "thresholdsStyle": {

  1791.                   "mode": "off"

  1792.                 }

  1793.               },

  1794.               "mappings": [],

  1795.               "thresholds": {

  1796.                 "mode": "absolute",

  1797.                 "steps": [

  1798.                   {

  1799.                     "color": "green"

  1800.                   },

  1801.                   {

  1802.                     "color": "red",

  1803.                     "value": 80

  1804.                   }

  1805.                 ]

  1806.               }

  1807.             },

  1808.             "overrides": []

  1809.           },

  1810.           "gridPos": {

  1811.             "h": 7,

  1812.             "w": 12,

  1813.             "x": 12,

  1814.             "y": 48

  1815.           },

  1816.           "id": 207,

  1817.           "options": {

  1818.             "legend": {

  1819.               "calcs": [],

  1820.               "displayMode": "list",

  1821.               "placement": "bottom",

  1822.               "showLegend": true

  1823.             },

  1824.             "tooltip": {

  1825.               "mode": "single",

  1826.               "sort": "none"

  1827.             }

  1828.           },

  1829.           "targets": [

  1830.             {

  1831.               "datasource": {

  1832.                 "type": "prometheus",

  1833.                 "uid": "${DS_PROMETHEUS}"

  1834.               },

  1835.               "exemplar": true,

  1836.               "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  1837.               "interval": "",

  1838.               "legendFormat": "{{job}}-{{index}} {{name}}",

  1839.               "refId": "A"

  1840.             }

  1841.           ],

  1842.           "title": "Threadpool activity",

  1843.           "type": "timeseries"

  1844.         }

  1845.       ],

  1846.       "targets": [

  1847.         {

  1848.           "datasource": {

  1849.             "type": "prometheus",

  1850.             "uid": "${DS_PROMETHEUS}"

  1851.           },

  1852.           "refId": "A"

  1853.         }

  1854.       ],

  1855.       "title": "Process info",

  1856.       "type": "row"

  1857.     },

  1858.     {

  1859.       "collapsed": true,

  1860.       "datasource": {

  1861.         "type": "prometheus",

  1862.         "uid": "${DS_PROMETHEUS}"

  1863.       },

  1864.       "gridPos": {

  1865.         "h": 1,

  1866.         "w": 24,

  1867.         "x": 0,

  1868.         "y": 27

  1869.       },

  1870.       "id": 56,

  1871.       "panels": [

  1872.         {

  1873.           "cards": {

  1874.             "cardPadding": -1,

  1875.             "cardRound": 0

  1876.           },

  1877.           "color": {

  1878.             "cardColor": "#b4ff00",

  1879.             "colorScale": "sqrt",

  1880.             "colorScheme": "interpolateInferno",

  1881.             "exponent": 0.5,

  1882.             "mode": "spectrum"

  1883.           },

  1884.           "dataFormat": "tsbuckets",

  1885.           "datasource": {

  1886.             "uid": "${DS_PROMETHEUS}"

  1887.           },

  1888.           "fieldConfig": {

  1889.             "defaults": {

  1890.               "custom": {

  1891.                 "hideFrom": {

  1892.                   "legend": false,

  1893.                   "tooltip": false,

  1894.                   "viz": false

  1895.                 },

  1896.                 "scaleDistribution": {

  1897.                   "type": "linear"

  1898.                 }

  1899.               }

  1900.             },

  1901.             "overrides": []

  1902.           },

  1903.           "gridPos": {

  1904.             "h": 9,

  1905.             "w": 12,

  1906.             "x": 0,

  1907.             "y": 28

  1908.           },

  1909.           "heatmap": {},

  1910.           "hideZeroBuckets": false,

  1911.           "highlightCards": true,

  1912.           "id": 85,

  1913.           "legend": {

  1914.             "show": false

  1915.           },

  1916.           "links": [],

  1917.           "options": {

  1918.             "calculate": false,

  1919.             "calculation": {},

  1920.             "cellGap": -1,

  1921.             "cellRadius": 0,

  1922.             "cellValues": {},

  1923.             "color": {

  1924.               "exponent": 0.5,

  1925.               "fill": "#b4ff00",

  1926.               "mode": "scheme",

  1927.               "reverse": false,

  1928.               "scale": "exponential",

  1929.               "scheme": "Inferno",

  1930.               "steps": 128

  1931.             },

  1932.             "exemplars": {

  1933.               "color": "rgba(255,0,255,0.7)"

  1934.             },

  1935.             "filterValues": {

  1936.               "le": 1e-9

  1937.             },

  1938.             "legend": {

  1939.               "show": false

  1940.             },

  1941.             "rowsFrame": {

  1942.               "layout": "auto"

  1943.             },

  1944.             "showValue": "never",

  1945.             "tooltip": {

  1946.               "show": true,

  1947.               "yHistogram": true

  1948.             },

  1949.             "yAxis": {

  1950.               "axisPlacement": "left",

  1951.               "reverse": false,

  1952.               "unit": "s"

  1953.             }

  1954.           },

  1955.           "pluginVersion": "9.2.2",

  1956.           "reverseYBuckets": false,

  1957.           "targets": [

  1958.             {

  1959.               "datasource": {

  1960.                 "uid": "${DS_PROMETHEUS}"

  1961.               },

  1962.               "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",

  1963.               "format": "heatmap",

  1964.               "intervalFactor": 1,

  1965.               "legendFormat": "{{le}}",

  1966.               "refId": "A"

  1967.             }

  1968.           ],

  1969.           "title": "Event Send Time (Including errors, across all workers)",

  1970.           "tooltip": {

  1971.             "show": true,

  1972.             "showHistogram": true

  1973.           },

  1974.           "type": "heatmap",

  1975.           "xAxis": {

  1976.             "show": true

  1977.           },

  1978.           "yAxis": {

  1979.             "format": "s",

  1980.             "logBase": 2,

  1981.             "show": true

  1982.           },

  1983.           "yBucketBound": "auto"

  1984.         },

  1985.         {

  1986.           "aliasColors": {},

  1987.           "bars": false,

  1988.           "dashLength": 10,

  1989.           "dashes": false,

  1990.           "datasource": {

  1991.             "uid": "${DS_PROMETHEUS}"

  1992.           },

  1993.           "description": "",

  1994.           "editable": true,

  1995.           "error": false,

  1996.           "fieldConfig": {

  1997.             "defaults": {

  1998.               "links": []

  1999.             },

  2000.             "overrides": []

  2001.           },

  2002.           "fill": 1,

  2003.           "fillGradient": 0,

  2004.           "grid": {},

  2005.           "gridPos": {

  2006.             "h": 9,

  2007.             "w": 12,

  2008.             "x": 12,

  2009.             "y": 28

  2010.           },

  2011.           "hiddenSeries": false,

  2012.           "id": 33,

  2013.           "legend": {

  2014.             "avg": false,

  2015.             "current": false,

  2016.             "max": false,

  2017.             "min": false,

  2018.             "show": false,

  2019.             "total": false,

  2020.             "values": false

  2021.           },

  2022.           "lines": true,

  2023.           "linewidth": 2,

  2024.           "links": [],

  2025.           "nullPointMode": "null",

  2026.           "options": {

  2027.             "alertThreshold": true

  2028.           },

  2029.           "paceLength": 10,

  2030.           "percentage": false,

  2031.           "pluginVersion": "9.2.2",

  2032.           "pointradius": 5,

  2033.           "points": false,

  2034.           "renderer": "flot",

  2035.           "seriesOverrides": [],

  2036.           "spaceLength": 10,

  2037.           "stack": false,

  2038.           "steppedLine": false,

  2039.           "targets": [

  2040.             {

  2041.               "datasource": {

  2042.                 "uid": "${DS_PROMETHEUS}"

  2043.               },

  2044.               "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)",

  2045.               "format": "time_series",

  2046.               "interval": "",

  2047.               "intervalFactor": 2,

  2048.               "legendFormat": "",

  2049.               "refId": "A",

  2050.               "step": 20,

  2051.               "target": ""

  2052.             }

  2053.           ],

  2054.           "thresholds": [],

  2055.           "timeRegions": [],

  2056.           "title": "Events Persisted (all workers)",

  2057.           "tooltip": {

  2058.             "shared": true,

  2059.             "sort": 0,

  2060.             "value_type": "cumulative"

  2061.           },

  2062.           "type": "graph",

  2063.           "xaxis": {

  2064.             "mode": "time",

  2065.             "show": true,

  2066.             "values": []

  2067.           },

  2068.           "yaxes": [

  2069.             {

  2070.               "$$hashKey": "object:102",

  2071.               "format": "hertz",

  2072.               "logBase": 1,

  2073.               "show": true

  2074.             },

  2075.             {

  2076.               "$$hashKey": "object:103",

  2077.               "format": "short",

  2078.               "logBase": 1,

  2079.               "show": true

  2080.             }

  2081.           ],

  2082.           "yaxis": {

  2083.             "align": false

  2084.           }

  2085.         },

  2086.         {

  2087.           "aliasColors": {},

  2088.           "bars": false,

  2089.           "dashLength": 10,

  2090.           "dashes": false,

  2091.           "datasource": {

  2092.             "uid": "${DS_PROMETHEUS}"

  2093.           },

  2094.           "decimals": 1,

  2095.           "fill": 1,

  2096.           "fillGradient": 0,

  2097.           "gridPos": {

  2098.             "h": 7,

  2099.             "w": 12,

  2100.             "x": 0,

  2101.             "y": 37

  2102.           },

  2103.           "hiddenSeries": false,

  2104.           "id": 40,

  2105.           "legend": {

  2106.             "avg": false,

  2107.             "current": false,

  2108.             "max": false,

  2109.             "min": false,

  2110.             "show": true,

  2111.             "total": false,

  2112.             "values": false

  2113.           },

  2114.           "lines": true,

  2115.           "linewidth": 1,

  2116.           "links": [],

  2117.           "nullPointMode": "null",

  2118.           "options": {

  2119.             "alertThreshold": true

  2120.           },

  2121.           "percentage": false,

  2122.           "pluginVersion": "9.2.2",

  2123.           "pointradius": 5,

  2124.           "points": false,

  2125.           "renderer": "flot",

  2126.           "seriesOverrides": [],

  2127.           "spaceLength": 10,

  2128.           "stack": false,

  2129.           "steppedLine": false,

  2130.           "targets": [

  2131.             {

  2132.               "datasource": {

  2133.                 "uid": "${DS_PROMETHEUS}"

  2134.               },

  2135.               "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  2136.               "format": "time_series",

  2137.               "intervalFactor": 2,

  2138.               "legendFormat": "{{type}}",

  2139.               "refId": "D"

  2140.             }

  2141.           ],

  2142.           "thresholds": [],

  2143.           "timeRegions": [],

  2144.           "title": "Events/s Local vs Remote",

  2145.           "tooltip": {

  2146.             "shared": true,

  2147.             "sort": 2,

  2148.             "value_type": "individual"

  2149.           },

  2150.           "type": "graph",

  2151.           "xaxis": {

  2152.             "mode": "time",

  2153.             "show": true,

  2154.             "values": []

  2155.           },

  2156.           "yaxes": [

  2157.             {

  2158.               "format": "hertz",

  2159.               "label": "",

  2160.               "logBase": 1,

  2161.               "min": "0",

  2162.               "show": true

  2163.             },

  2164.             {

  2165.               "format": "short",

  2166.               "logBase": 1,

  2167.               "show": true

  2168.             }

  2169.           ],

  2170.           "yaxis": {

  2171.             "align": false

  2172.           }

  2173.         },

  2174.         {

  2175.           "aliasColors": {},

  2176.           "bars": false,

  2177.           "dashLength": 10,

  2178.           "dashes": false,

  2179.           "datasource": {

  2180.             "uid": "${DS_PROMETHEUS}"

  2181.           },

  2182.           "decimals": 1,

  2183.           "fill": 1,

  2184.           "fillGradient": 0,

  2185.           "gridPos": {

  2186.             "h": 7,

  2187.             "w": 12,

  2188.             "x": 12,

  2189.             "y": 37

  2190.           },

  2191.           "hiddenSeries": false,

  2192.           "id": 46,

  2193.           "legend": {

  2194.             "avg": false,

  2195.             "current": false,

  2196.             "max": false,

  2197.             "min": false,

  2198.             "show": true,

  2199.             "total": false,

  2200.             "values": false

  2201.           },

  2202.           "lines": true,

  2203.           "linewidth": 1,

  2204.           "links": [],

  2205.           "nullPointMode": "null",

  2206.           "options": {

  2207.             "alertThreshold": true

  2208.           },

  2209.           "percentage": false,

  2210.           "pluginVersion": "9.2.2",

  2211.           "pointradius": 5,

  2212.           "points": false,

  2213.           "renderer": "flot",

  2214.           "seriesOverrides": [],

  2215.           "spaceLength": 10,

  2216.           "stack": false,

  2217.           "steppedLine": false,

  2218.           "targets": [

  2219.             {

  2220.               "datasource": {

  2221.                 "uid": "${DS_PROMETHEUS}"

  2222.               },

  2223.               "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  2224.               "format": "time_series",

  2225.               "instant": false,

  2226.               "intervalFactor": 2,

  2227.               "legendFormat": "{{type}}",

  2228.               "refId": "A",

  2229.               "step": 20

  2230.             }

  2231.           ],

  2232.           "thresholds": [],

  2233.           "timeRegions": [],

  2234.           "title": "Events/s by Type",

  2235.           "tooltip": {

  2236.             "shared": false,

  2237.             "sort": 2,

  2238.             "value_type": "individual"

  2239.           },

  2240.           "type": "graph",

  2241.           "xaxis": {

  2242.             "mode": "time",

  2243.             "show": true,

  2244.             "values": []

  2245.           },

  2246.           "yaxes": [

  2247.             {

  2248.               "format": "hertz",

  2249.               "logBase": 1,

  2250.               "min": "0",

  2251.               "show": true

  2252.             },

  2253.             {

  2254.               "format": "short",

  2255.               "logBase": 1,

  2256.               "show": true

  2257.             }

  2258.           ],

  2259.           "yaxis": {

  2260.             "align": false

  2261.           }

  2262.         },

  2263.         {

  2264.           "aliasColors": {

  2265.             "irc-freenode (local)": "#EAB839"

  2266.           },

  2267.           "bars": false,

  2268.           "dashLength": 10,

  2269.           "dashes": false,

  2270.           "datasource": {

  2271.             "uid": "${DS_PROMETHEUS}"

  2272.           },

  2273.           "decimals": 1,

  2274.           "fill": 1,

  2275.           "fillGradient": 0,

  2276.           "gridPos": {

  2277.             "h": 7,

  2278.             "w": 12,

  2279.             "x": 0,

  2280.             "y": 44

  2281.           },

  2282.           "hiddenSeries": false,

  2283.           "id": 44,

  2284.           "legend": {

  2285.             "alignAsTable": true,

  2286.             "avg": false,

  2287.             "current": false,

  2288.             "hideEmpty": true,

  2289.             "hideZero": true,

  2290.             "max": false,

  2291.             "min": false,

  2292.             "show": true,

  2293.             "total": false,

  2294.             "values": false

  2295.           },

  2296.           "lines": true,

  2297.           "linewidth": 1,

  2298.           "links": [],

  2299.           "nullPointMode": "null",

  2300.           "options": {

  2301.             "alertThreshold": true

  2302.           },

  2303.           "percentage": false,

  2304.           "pluginVersion": "9.2.2",

  2305.           "pointradius": 5,

  2306.           "points": false,

  2307.           "renderer": "flot",

  2308.           "seriesOverrides": [],

  2309.           "spaceLength": 10,

  2310.           "stack": false,

  2311.           "steppedLine": false,

  2312.           "targets": [

  2313.             {

  2314.               "datasource": {

  2315.                 "uid": "${DS_PROMETHEUS}"

  2316.               },

  2317.               "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  2318.               "format": "time_series",

  2319.               "intervalFactor": 2,

  2320.               "legendFormat": "{{origin_entity}} ({{origin_type}})",

  2321.               "refId": "A",

  2322.               "step": 20

  2323.             }

  2324.           ],

  2325.           "thresholds": [],

  2326.           "timeRegions": [],

  2327.           "title": "Events/s by Origin",

  2328.           "tooltip": {

  2329.             "shared": false,

  2330.             "sort": 2,

  2331.             "value_type": "individual"

  2332.           },

  2333.           "type": "graph",

  2334.           "xaxis": {

  2335.             "mode": "time",

  2336.             "show": true,

  2337.             "values": []

  2338.           },

  2339.           "yaxes": [

  2340.             {

  2341.               "format": "hertz",

  2342.               "logBase": 1,

  2343.               "min": "0",

  2344.               "show": true

  2345.             },

  2346.             {

  2347.               "format": "short",

  2348.               "logBase": 1,

  2349.               "show": true

  2350.             }

  2351.           ],

  2352.           "yaxis": {

  2353.             "align": false

  2354.           }

  2355.         },

  2356.         {

  2357.           "aliasColors": {},

  2358.           "bars": false,

  2359.           "dashLength": 10,

  2360.           "dashes": false,

  2361.           "datasource": {

  2362.             "type": "prometheus",

  2363.             "uid": "${DS_PROMETHEUS}"

  2364.           },

  2365.           "decimals": 1,

  2366.           "fill": 1,

  2367.           "fillGradient": 0,

  2368.           "gridPos": {

  2369.             "h": 7,

  2370.             "w": 12,

  2371.             "x": 12,

  2372.             "y": 44

  2373.           },

  2374.           "hiddenSeries": false,

  2375.           "id": 45,

  2376.           "legend": {

  2377.             "alignAsTable": true,

  2378.             "avg": false,

  2379.             "current": false,

  2380.             "hideEmpty": true,

  2381.             "hideZero": true,

  2382.             "max": false,

  2383.             "min": false,

  2384.             "show": true,

  2385.             "total": false,

  2386.             "values": false

  2387.           },

  2388.           "lines": true,

  2389.           "linewidth": 1,

  2390.           "links": [],

  2391.           "nullPointMode": "null",

  2392.           "options": {

  2393.             "alertThreshold": true

  2394.           },

  2395.           "percentage": false,

  2396.           "pluginVersion": "9.2.2",

  2397.           "pointradius": 5,

  2398.           "points": false,

  2399.           "renderer": "flot",

  2400.           "seriesOverrides": [],

  2401.           "spaceLength": 10,

  2402.           "stack": false,

  2403.           "steppedLine": false,

  2404.           "targets": [

  2405.             {

  2406.               "datasource": {

  2407.                 "uid": "${DS_PROMETHEUS}"

  2408.               },

  2409.               "editorMode": "code",

  2410.               "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",

  2411.               "format": "time_series",

  2412.               "intervalFactor": 2,

  2413.               "legendFormat": "{{origin_entity}} ({{origin_type}})",

  2414.               "range": true,

  2415.               "refId": "A",

  2416.               "step": 20

  2417.             }

  2418.           ],

  2419.           "thresholds": [],

  2420.           "timeRegions": [],

  2421.           "title": "Memberships/s by Origin",

  2422.           "tooltip": {

  2423.             "shared": true,

  2424.             "sort": 2,

  2425.             "value_type": "individual"

  2426.           },

  2427.           "type": "graph",

  2428.           "xaxis": {

  2429.             "mode": "time",

  2430.             "show": true,

  2431.             "values": []

  2432.           },

  2433.           "yaxes": [

  2434.             {

  2435.               "$$hashKey": "object:232",

  2436.               "format": "hertz",

  2437.               "logBase": 1,

  2438.               "min": "0",

  2439.               "show": true

  2440.             },

  2441.             {

  2442.               "$$hashKey": "object:233",

  2443.               "format": "short",

  2444.               "logBase": 1,

  2445.               "show": true

  2446.             }

  2447.           ],

  2448.           "yaxis": {

  2449.             "align": false

  2450.           }

  2451.         },

  2452.         {

  2453.           "aliasColors": {},

  2454.           "bars": false,

  2455.           "dashLength": 10,

  2456.           "dashes": false,

  2457.           "datasource": {

  2458.             "type": "prometheus",

  2459.             "uid": "${DS_PROMETHEUS}"

  2460.           },

  2461.           "fieldConfig": {

  2462.             "defaults": {

  2463.               "links": []

  2464.             },

  2465.             "overrides": []

  2466.           },

  2467.           "fill": 0,

  2468.           "fillGradient": 0,

  2469.           "gridPos": {

  2470.             "h": 9,

  2471.             "w": 12,

  2472.             "x": 0,

  2473.             "y": 51

  2474.           },

  2475.           "hiddenSeries": false,

  2476.           "id": 118,

  2477.           "legend": {

  2478.             "avg": false,

  2479.             "current": false,

  2480.             "max": false,

  2481.             "min": false,

  2482.             "show": true,

  2483.             "total": false,

  2484.             "values": false

  2485.           },

  2486.           "lines": true,

  2487.           "linewidth": 1,

  2488.           "links": [],

  2489.           "nullPointMode": "null",

  2490.           "options": {

  2491.             "alertThreshold": true

  2492.           },

  2493.           "paceLength": 10,

  2494.           "percentage": false,

  2495.           "pluginVersion": "9.2.2",

  2496.           "pointradius": 5,

  2497.           "points": false,

  2498.           "renderer": "flot",

  2499.           "repeatDirection": "h",

  2500.           "seriesOverrides": [

  2501.             {

  2502.               "$$hashKey": "object:316",

  2503.               "alias": "mean",

  2504.               "linewidth": 2

  2505.             }

  2506.           ],

  2507.           "spaceLength": 10,

  2508.           "stack": false,

  2509.           "steppedLine": false,

  2510.           "targets": [

  2511.             {

  2512.               "datasource": {

  2513.                 "type": "prometheus",

  2514.                 "uid": "${DS_PROMETHEUS}"

  2515.               },

  2516.               "exemplar": true,

  2517.               "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",

  2518.               "format": "time_series",

  2519.               "interval": "",

  2520.               "intervalFactor": 1,

  2521.               "legendFormat": "{{job}}-{{index}} 99%",

  2522.               "refId": "A"

  2523.             },

  2524.             {

  2525.               "datasource": {

  2526.                 "type": "prometheus",

  2527.                 "uid": "${DS_PROMETHEUS}"

  2528.               },

  2529.               "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",

  2530.               "format": "time_series",

  2531.               "interval": "",

  2532.               "intervalFactor": 1,

  2533.               "legendFormat": "{{job}}-{{index}} 95%",

  2534.               "refId": "B"

  2535.             },

  2536.             {

  2537.               "datasource": {

  2538.                 "type": "prometheus",

  2539.                 "uid": "${DS_PROMETHEUS}"

  2540.               },

  2541.               "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",

  2542.               "format": "time_series",

  2543.               "intervalFactor": 1,

  2544.               "legendFormat": "{{job}}-{{index}} 90%",

  2545.               "refId": "C"

  2546.             },

  2547.             {

  2548.               "datasource": {

  2549.                 "type": "prometheus",

  2550.                 "uid": "${DS_PROMETHEUS}"

  2551.               },

  2552.               "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",

  2553.               "format": "time_series",

  2554.               "intervalFactor": 1,

  2555.               "legendFormat": "{{job}}-{{index}} 50%",

  2556.               "refId": "D"

  2557.             },

  2558.             {

  2559.               "datasource": {

  2560.                 "type": "prometheus",

  2561.                 "uid": "${DS_PROMETHEUS}"

  2562.               },

  2563.               "editorMode": "code",

  2564.               "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",

  2565.               "format": "time_series",

  2566.               "intervalFactor": 1,

  2567.               "legendFormat": "{{job}}-{{index}} mean",

  2568.               "range": true,

  2569.               "refId": "E"

  2570.             }

  2571.           ],

  2572.           "thresholds": [],

  2573.           "timeRegions": [],

  2574.           "title": "Event send time quantiles by worker",

  2575.           "tooltip": {

  2576.             "shared": true,

  2577.             "sort": 0,

  2578.             "value_type": "individual"

  2579.           },

  2580.           "type": "graph",

  2581.           "xaxis": {

  2582.             "mode": "time",

  2583.             "show": true,

  2584.             "values": []

  2585.           },

  2586.           "yaxes": [

  2587.             {

  2588.               "$$hashKey": "object:263",

  2589.               "format": "s",

  2590.               "logBase": 1,

  2591.               "show": true

  2592.             },

  2593.             {

  2594.               "$$hashKey": "object:264",

  2595.               "format": "short",

  2596.               "logBase": 1,

  2597.               "show": true

  2598.             }

  2599.           ],

  2600.           "yaxis": {

  2601.             "align": false

  2602.           }

  2603.         },

  2604.         {

  2605.           "datasource": {

  2606.             "type": "prometheus",

  2607.             "uid": "${DS_PROMETHEUS}"

  2608.           },

  2609.           "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n",

  2610.           "fieldConfig": {

  2611.             "defaults": {

  2612.               "color": {

  2613.                 "mode": "palette-classic"

  2614.               },

  2615.               "custom": {

  2616.                 "axisCenteredZero": false,

  2617.                 "axisColorMode": "text",

  2618.                 "axisLabel": "",

  2619.                 "axisPlacement": "auto",

  2620.                 "barAlignment": 0,

  2621.                 "drawStyle": "line",

  2622.                 "fillOpacity": 30,

  2623.                 "gradientMode": "none",

  2624.                 "hideFrom": {

  2625.                   "legend": false,

  2626.                   "tooltip": false,

  2627.                   "viz": false

  2628.                 },

  2629.                 "lineInterpolation": "linear",

  2630.                 "lineWidth": 1,

  2631.                 "pointSize": 5,

  2632.                 "scaleDistribution": {

  2633.                   "type": "linear"

  2634.                 },

  2635.                 "showPoints": "auto",

  2636.                 "spanNulls": false,

  2637.                 "stacking": {

  2638.                   "group": "A",

  2639.                   "mode": "normal"

  2640.                 },

  2641.                 "thresholdsStyle": {

  2642.                   "mode": "off"

  2643.                 }

  2644.               },

  2645.               "mappings": [],

  2646.               "thresholds": {

  2647.                 "mode": "absolute",

  2648.                 "steps": [

  2649.                   {

  2650.                     "color": "green"

  2651.                   },

  2652.                   {

  2653.                     "color": "red",

  2654.                     "value": 80

  2655.                   }

  2656.                 ]

  2657.               },

  2658.               "unit": "s/s"

  2659.             },

  2660.             "overrides": []

  2661.           },

  2662.           "gridPos": {

  2663.             "h": 9,

  2664.             "w": 12,

  2665.             "x": 12,

  2666.             "y": 51

  2667.           },

  2668.           "id": 222,

  2669.           "options": {

  2670.             "legend": {

  2671.               "calcs": [],

  2672.               "displayMode": "list",

  2673.               "placement": "bottom",

  2674.               "showLegend": false

  2675.             },

  2676.             "tooltip": {

  2677.               "mode": "multi",

  2678.               "sort": "none"

  2679.             }

  2680.           },

  2681.           "targets": [

  2682.             {

  2683.               "datasource": {

  2684.                 "type": "prometheus",

  2685.                 "uid": "${DS_PROMETHEUS}"

  2686.               },

  2687.               "exemplar": false,

  2688.               "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",

  2689.               "format": "time_series",

  2690.               "hide": false,

  2691.               "instant": false,

  2692.               "interval": "",

  2693.               "legendFormat": "DB time",

  2694.               "refId": "B"

  2695.             },

  2696.             {

  2697.               "datasource": {

  2698.                 "type": "prometheus",

  2699.                 "uid": "${DS_PROMETHEUS}"

  2700.               },

  2701.               "exemplar": false,

  2702.               "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",

  2703.               "format": "time_series",

  2704.               "hide": false,

  2705.               "instant": false,

  2706.               "interval": "",

  2707.               "legendFormat": "CPU time",

  2708.               "refId": "C"

  2709.             }

  2710.           ],

  2711.           "title": "Stateres worst-case",

  2712.           "type": "timeseries"

  2713.         }

  2714.       ],

  2715.       "targets": [

  2716.         {

  2717.           "datasource": {

  2718.             "type": "prometheus",

  2719.             "uid": "${DS_PROMETHEUS}"

  2720.           },

  2721.           "refId": "A"

  2722.         }

  2723.       ],

  2724.       "title": "Event persistence",

  2725.       "type": "row"

  2726.     },

  2727.     {

  2728.       "collapsed": true,

  2729.       "datasource": {

  2730.         "type": "prometheus",

  2731.         "uid": "${DS_PROMETHEUS}"

  2732.       },

  2733.       "gridPos": {

  2734.         "h": 1,

  2735.         "w": 24,

  2736.         "x": 0,

  2737.         "y": 28

  2738.       },

  2739.       "id": 57,

  2740.       "panels": [

  2741.         {

  2742.           "aliasColors": {},

  2743.           "bars": false,

  2744.           "dashLength": 10,

  2745.           "dashes": false,

  2746.           "datasource": {

  2747.             "type": "prometheus",

  2748.             "uid": "${DS_PROMETHEUS}"

  2749.           },

  2750.           "editable": true,

  2751.           "error": false,

  2752.           "fieldConfig": {

  2753.             "defaults": {

  2754.               "links": []

  2755.             },

  2756.             "overrides": []

  2757.           },

  2758.           "fill": 2,

  2759.           "fillGradient": 0,

  2760.           "grid": {},

  2761.           "gridPos": {

  2762.             "h": 8,

  2763.             "w": 12,

  2764.             "x": 0,

  2765.             "y": 29

  2766.           },

  2767.           "hiddenSeries": false,

  2768.           "id": 4,

  2769.           "legend": {

  2770.             "alignAsTable": true,

  2771.             "avg": false,

  2772.             "current": false,

  2773.             "hideEmpty": false,

  2774.             "hideZero": true,

  2775.             "max": false,

  2776.             "min": false,

  2777.             "rightSide": false,

  2778.             "show": true,

  2779.             "total": false,

  2780.             "values": false

  2781.           },

  2782.           "lines": true,

  2783.           "linewidth": 1,

  2784.           "links": [],

  2785.           "nullPointMode": "null",

  2786.           "options": {

  2787.             "alertThreshold": true

  2788.           },

  2789.           "percentage": false,

  2790.           "pluginVersion": "9.2.2",

  2791.           "pointradius": 5,

  2792.           "points": false,

  2793.           "renderer": "flot",

  2794.           "seriesOverrides": [],

  2795.           "spaceLength": 10,

  2796.           "stack": false,

  2797.           "steppedLine": false,

  2798.           "targets": [

  2799.             {

  2800.               "datasource": {

  2801.                 "uid": "${DS_PROMETHEUS}"

  2802.               },

  2803.               "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  2804.               "format": "time_series",

  2805.               "interval": "",

  2806.               "intervalFactor": 2,

  2807.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",

  2808.               "refId": "A",

  2809.               "step": 20

  2810.             }

  2811.           ],

  2812.           "thresholds": [

  2813.             {

  2814.               "$$hashKey": "object:234",

  2815.               "colorMode": "custom",

  2816.               "fill": true,

  2817.               "fillColor": "rgba(216, 200, 27, 0.27)",

  2818.               "op": "gt",

  2819.               "value": 100,

  2820.               "yaxis": "left"

  2821.             },

  2822.             {

  2823.               "$$hashKey": "object:235",

  2824.               "colorMode": "custom",

  2825.               "fill": true,

  2826.               "fillColor": "rgba(234, 112, 112, 0.22)",

  2827.               "op": "gt",

  2828.               "value": 250,

  2829.               "yaxis": "left"

  2830.             }

  2831.           ],

  2832.           "timeRegions": [],

  2833.           "title": "Request Count by arrival time",

  2834.           "tooltip": {

  2835.             "shared": false,

  2836.             "sort": 2,

  2837.             "value_type": "individual"

  2838.           },

  2839.           "type": "graph",

  2840.           "xaxis": {

  2841.             "mode": "time",

  2842.             "show": true,

  2843.             "values": []

  2844.           },

  2845.           "yaxes": [

  2846.             {

  2847.               "$$hashKey": "object:206",

  2848.               "format": "hertz",

  2849.               "logBase": 1,

  2850.               "show": true

  2851.             },

  2852.             {

  2853.               "$$hashKey": "object:207",

  2854.               "format": "short",

  2855.               "logBase": 1,

  2856.               "show": true

  2857.             }

  2858.           ],

  2859.           "yaxis": {

  2860.             "align": false

  2861.           }

  2862.         },

  2863.         {

  2864.           "aliasColors": {},

  2865.           "bars": false,

  2866.           "dashLength": 10,

  2867.           "dashes": false,

  2868.           "datasource": {

  2869.             "type": "prometheus",

  2870.             "uid": "${DS_PROMETHEUS}"

  2871.           },

  2872.           "editable": true,

  2873.           "error": false,

  2874.           "fieldConfig": {

  2875.             "defaults": {

  2876.               "links": []

  2877.             },

  2878.             "overrides": []

  2879.           },

  2880.           "fill": 1,

  2881.           "fillGradient": 0,

  2882.           "grid": {},

  2883.           "gridPos": {

  2884.             "h": 8,

  2885.             "w": 12,

  2886.             "x": 12,

  2887.             "y": 29

  2888.           },

  2889.           "hiddenSeries": false,

  2890.           "id": 32,

  2891.           "legend": {

  2892.             "avg": false,

  2893.             "current": false,

  2894.             "max": false,

  2895.             "min": false,

  2896.             "show": true,

  2897.             "total": false,

  2898.             "values": false

  2899.           },

  2900.           "lines": true,

  2901.           "linewidth": 2,

  2902.           "links": [],

  2903.           "nullPointMode": "null",

  2904.           "options": {

  2905.             "alertThreshold": true

  2906.           },

  2907.           "percentage": false,

  2908.           "pluginVersion": "9.2.2",

  2909.           "pointradius": 5,

  2910.           "points": false,

  2911.           "renderer": "flot",

  2912.           "seriesOverrides": [],

  2913.           "spaceLength": 10,

  2914.           "stack": false,

  2915.           "steppedLine": false,

  2916.           "targets": [

  2917.             {

  2918.               "datasource": {

  2919.                 "uid": "${DS_PROMETHEUS}"

  2920.               },

  2921.               "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",

  2922.               "format": "time_series",

  2923.               "intervalFactor": 2,

  2924.               "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",

  2925.               "refId": "A",

  2926.               "step": 20,

  2927.               "target": ""

  2928.             }

  2929.           ],

  2930.           "thresholds": [],

  2931.           "timeRegions": [],

  2932.           "title": "Top 10 Request Counts",

  2933.           "tooltip": {

  2934.             "shared": false,

  2935.             "sort": 2,

  2936.             "value_type": "cumulative"

  2937.           },

  2938.           "type": "graph",

  2939.           "xaxis": {

  2940.             "mode": "time",

  2941.             "show": true,

  2942.             "values": []

  2943.           },

  2944.           "yaxes": [

  2945.             {

  2946.               "$$hashKey": "object:305",

  2947.               "format": "hertz",

  2948.               "logBase": 1,

  2949.               "show": true

  2950.             },

  2951.             {

  2952.               "$$hashKey": "object:306",

  2953.               "format": "short",

  2954.               "logBase": 1,

  2955.               "show": true

  2956.             }

  2957.           ],

  2958.           "yaxis": {

  2959.             "align": false

  2960.           }

  2961.         },

  2962.         {

  2963.           "aliasColors": {},

  2964.           "bars": false,

  2965.           "dashLength": 10,

  2966.           "dashes": false,

  2967.           "datasource": {

  2968.             "type": "prometheus",

  2969.             "uid": "${DS_PROMETHEUS}"

  2970.           },

  2971.           "editable": true,

  2972.           "error": false,

  2973.           "fieldConfig": {

  2974.             "defaults": {

  2975.               "links": []

  2976.             },

  2977.             "overrides": []

  2978.           },

  2979.           "fill": 2,

  2980.           "fillGradient": 0,

  2981.           "grid": {},

  2982.           "gridPos": {

  2983.             "h": 8,

  2984.             "w": 12,

  2985.             "x": 0,

  2986.             "y": 37

  2987.           },

  2988.           "hiddenSeries": false,

  2989.           "id": 139,

  2990.           "legend": {

  2991.             "alignAsTable": true,

  2992.             "avg": false,

  2993.             "current": false,

  2994.             "hideEmpty": false,

  2995.             "hideZero": true,

  2996.             "max": false,

  2997.             "min": false,

  2998.             "rightSide": false,

  2999.             "show": true,

  3000.             "total": false,

  3001.             "values": false

  3002.           },

  3003.           "lines": true,

  3004.           "linewidth": 1,

  3005.           "links": [],

  3006.           "nullPointMode": "null",

  3007.           "options": {

  3008.             "alertThreshold": true

  3009.           },

  3010.           "percentage": false,

  3011.           "pluginVersion": "9.2.2",

  3012.           "pointradius": 5,

  3013.           "points": false,

  3014.           "renderer": "flot",

  3015.           "seriesOverrides": [],

  3016.           "spaceLength": 10,

  3017.           "stack": false,

  3018.           "steppedLine": false,

  3019.           "targets": [

  3020.             {

  3021.               "datasource": {

  3022.                 "uid": "${DS_PROMETHEUS}"

  3023.               },

  3024.               "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  3025.               "format": "time_series",

  3026.               "interval": "",

  3027.               "intervalFactor": 1,

  3028.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",

  3029.               "refId": "A",

  3030.               "step": 20

  3031.             }

  3032.           ],

  3033.           "thresholds": [

  3034.             {

  3035.               "$$hashKey": "object:135",

  3036.               "colorMode": "custom",

  3037.               "fill": true,

  3038.               "fillColor": "rgba(216, 200, 27, 0.27)",

  3039.               "op": "gt",

  3040.               "value": 100,

  3041.               "yaxis": "left"

  3042.             },

  3043.             {

  3044.               "$$hashKey": "object:136",

  3045.               "colorMode": "custom",

  3046.               "fill": true,

  3047.               "fillColor": "rgba(234, 112, 112, 0.22)",

  3048.               "op": "gt",

  3049.               "value": 250,

  3050.               "yaxis": "left"

  3051.             }

  3052.           ],

  3053.           "timeRegions": [],

  3054.           "title": "Total CPU Usage by Endpoint",

  3055.           "tooltip": {

  3056.             "shared": false,

  3057.             "sort": 2,

  3058.             "value_type": "individual"

  3059.           },

  3060.           "type": "graph",

  3061.           "xaxis": {

  3062.             "mode": "time",

  3063.             "show": true,

  3064.             "values": []

  3065.           },

  3066.           "yaxes": [

  3067.             {

  3068.               "$$hashKey": "object:107",

  3069.               "format": "percentunit",

  3070.               "logBase": 1,

  3071.               "show": true

  3072.             },

  3073.             {

  3074.               "$$hashKey": "object:108",

  3075.               "format": "short",

  3076.               "logBase": 1,

  3077.               "show": true

  3078.             }

  3079.           ],

  3080.           "yaxis": {

  3081.             "align": false

  3082.           }

  3083.         },

  3084.         {

  3085.           "aliasColors": {},

  3086.           "bars": false,

  3087.           "dashLength": 10,

  3088.           "dashes": false,

  3089.           "datasource": {

  3090.             "type": "prometheus",

  3091.             "uid": "${DS_PROMETHEUS}"

  3092.           },

  3093.           "editable": true,

  3094.           "error": false,

  3095.           "fieldConfig": {

  3096.             "defaults": {

  3097.               "links": []

  3098.             },

  3099.             "overrides": []

  3100.           },

  3101.           "fill": 0,

  3102.           "fillGradient": 0,

  3103.           "grid": {},

  3104.           "gridPos": {

  3105.             "h": 8,

  3106.             "w": 12,

  3107.             "x": 12,

  3108.             "y": 37

  3109.           },

  3110.           "hiddenSeries": false,

  3111.           "id": 52,

  3112.           "legend": {

  3113.             "alignAsTable": true,

  3114.             "avg": false,

  3115.             "current": false,

  3116.             "hideEmpty": false,

  3117.             "hideZero": true,

  3118.             "max": false,

  3119.             "min": false,

  3120.             "rightSide": false,

  3121.             "show": true,

  3122.             "total": false,

  3123.             "values": false

  3124.           },

  3125.           "lines": true,

  3126.           "linewidth": 1,

  3127.           "links": [],

  3128.           "nullPointMode": "null",

  3129.           "options": {

  3130.             "alertThreshold": true

  3131.           },

  3132.           "percentage": false,

  3133.           "pluginVersion": "9.2.2",

  3134.           "pointradius": 5,

  3135.           "points": false,

  3136.           "renderer": "flot",

  3137.           "seriesOverrides": [],

  3138.           "spaceLength": 10,

  3139.           "stack": false,

  3140.           "steppedLine": false,

  3141.           "targets": [

  3142.             {

  3143.               "datasource": {

  3144.                 "uid": "${DS_PROMETHEUS}"

  3145.               },

  3146.               "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  3147.               "format": "time_series",

  3148.               "interval": "",

  3149.               "intervalFactor": 1,

  3150.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",

  3151.               "refId": "A",

  3152.               "step": 20

  3153.             }

  3154.           ],

  3155.           "thresholds": [

  3156.             {

  3157.               "$$hashKey": "object:417",

  3158.               "colorMode": "custom",

  3159.               "fill": true,

  3160.               "fillColor": "rgba(216, 200, 27, 0.27)",

  3161.               "op": "gt",

  3162.               "value": 100,

  3163.               "yaxis": "left"

  3164.             },

  3165.             {

  3166.               "$$hashKey": "object:418",

  3167.               "colorMode": "custom",

  3168.               "fill": true,

  3169.               "fillColor": "rgba(234, 112, 112, 0.22)",

  3170.               "op": "gt",

  3171.               "value": 250,

  3172.               "yaxis": "left"

  3173.             }

  3174.           ],

  3175.           "timeRegions": [],

  3176.           "title": "Average CPU Usage by Endpoint",

  3177.           "tooltip": {

  3178.             "shared": false,

  3179.             "sort": 2,

  3180.             "value_type": "individual"

  3181.           },

  3182.           "type": "graph",

  3183.           "xaxis": {

  3184.             "mode": "time",

  3185.             "show": true,

  3186.             "values": []

  3187.           },

  3188.           "yaxes": [

  3189.             {

  3190.               "$$hashKey": "object:389",

  3191.               "format": "s",

  3192.               "logBase": 1,

  3193.               "show": true

  3194.             },

  3195.             {

  3196.               "$$hashKey": "object:390",

  3197.               "format": "short",

  3198.               "logBase": 1,

  3199.               "show": true

  3200.             }

  3201.           ],

  3202.           "yaxis": {

  3203.             "align": false

  3204.           }

  3205.         },

  3206.         {

  3207.           "aliasColors": {},

  3208.           "bars": false,

  3209.           "dashLength": 10,

  3210.           "dashes": false,

  3211.           "datasource": {

  3212.             "type": "prometheus",

  3213.             "uid": "${DS_PROMETHEUS}"

  3214.           },

  3215.           "editable": true,

  3216.           "error": false,

  3217.           "fieldConfig": {

  3218.             "defaults": {

  3219.               "links": []

  3220.             },

  3221.             "overrides": []

  3222.           },

  3223.           "fill": 1,

  3224.           "fillGradient": 0,

  3225.           "grid": {},

  3226.           "gridPos": {

  3227.             "h": 8,

  3228.             "w": 12,

  3229.             "x": 0,

  3230.             "y": 45

  3231.           },

  3232.           "hiddenSeries": false,

  3233.           "id": 7,

  3234.           "legend": {

  3235.             "alignAsTable": true,

  3236.             "avg": false,

  3237.             "current": false,

  3238.             "hideEmpty": true,

  3239.             "hideZero": true,

  3240.             "max": false,

  3241.             "min": false,

  3242.             "show": true,

  3243.             "total": false,

  3244.             "values": false

  3245.           },

  3246.           "lines": true,

  3247.           "linewidth": 1,

  3248.           "links": [],

  3249.           "nullPointMode": "null",

  3250.           "options": {

  3251.             "alertThreshold": true

  3252.           },

  3253.           "percentage": false,

  3254.           "pluginVersion": "9.2.2",

  3255.           "pointradius": 5,

  3256.           "points": false,

  3257.           "renderer": "flot",

  3258.           "seriesOverrides": [],

  3259.           "spaceLength": 10,

  3260.           "stack": false,

  3261.           "steppedLine": false,

  3262.           "targets": [

  3263.             {

  3264.               "datasource": {

  3265.                 "uid": "${DS_PROMETHEUS}"

  3266.               },

  3267.               "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  3268.               "format": "time_series",

  3269.               "interval": "",

  3270.               "intervalFactor": 1,

  3271.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",

  3272.               "refId": "A",

  3273.               "step": 20

  3274.             }

  3275.           ],

  3276.           "thresholds": [],

  3277.           "timeRegions": [],

  3278.           "title": "DB Usage by endpoint",

  3279.           "tooltip": {

  3280.             "shared": false,

  3281.             "sort": 2,

  3282.             "value_type": "cumulative"

  3283.           },

  3284.           "type": "graph",

  3285.           "xaxis": {

  3286.             "mode": "time",

  3287.             "show": true,

  3288.             "values": []

  3289.           },

  3290.           "yaxes": [

  3291.             {

  3292.               "$$hashKey": "object:488",

  3293.               "format": "percentunit",

  3294.               "logBase": 1,

  3295.               "show": true

  3296.             },

  3297.             {

  3298.               "$$hashKey": "object:489",

  3299.               "format": "short",

  3300.               "logBase": 1,

  3301.               "show": true

  3302.             }

  3303.           ],

  3304.           "yaxis": {

  3305.             "align": false

  3306.           }

  3307.         },

  3308.         {

  3309.           "aliasColors": {},

  3310.           "bars": false,

  3311.           "dashLength": 10,

  3312.           "dashes": false,

  3313.           "datasource": {

  3314.             "uid": "${DS_PROMETHEUS}"

  3315.           },

  3316.           "editable": true,

  3317.           "error": false,

  3318.           "fieldConfig": {

  3319.             "defaults": {

  3320.               "links": []

  3321.             },

  3322.             "overrides": []

  3323.           },

  3324.           "fill": 2,

  3325.           "fillGradient": 0,

  3326.           "grid": {},

  3327.           "gridPos": {

  3328.             "h": 8,

  3329.             "w": 12,

  3330.             "x": 12,

  3331.             "y": 45

  3332.           },

  3333.           "hiddenSeries": false,

  3334.           "id": 47,

  3335.           "legend": {

  3336.             "alignAsTable": true,

  3337.             "avg": true,

  3338.             "current": false,

  3339.             "hideEmpty": false,

  3340.             "hideZero": true,

  3341.             "max": true,

  3342.             "min": false,

  3343.             "rightSide": false,

  3344.             "show": true,

  3345.             "total": false,

  3346.             "values": true

  3347.           },

  3348.           "lines": true,

  3349.           "linewidth": 1,

  3350.           "links": [],

  3351.           "nullPointMode": "null",

  3352.           "options": {

  3353.             "alertThreshold": true

  3354.           },

  3355.           "percentage": false,

  3356.           "pluginVersion": "9.2.2",

  3357.           "pointradius": 5,

  3358.           "points": false,

  3359.           "renderer": "flot",

  3360.           "seriesOverrides": [],

  3361.           "spaceLength": 10,

  3362.           "stack": false,

  3363.           "steppedLine": false,

  3364.           "targets": [

  3365.             {

  3366.               "datasource": {

  3367.                 "uid": "${DS_PROMETHEUS}"

  3368.               },

  3369.               "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",

  3370.               "format": "time_series",

  3371.               "hide": false,

  3372.               "interval": "",

  3373.               "intervalFactor": 1,

  3374.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",

  3375.               "refId": "A",

  3376.               "step": 20

  3377.             }

  3378.           ],

  3379.           "thresholds": [],

  3380.           "timeRegions": [],

  3381.           "title": "Non-sync avg response time",

  3382.           "tooltip": {

  3383.             "shared": false,

  3384.             "sort": 2,

  3385.             "value_type": "individual"

  3386.           },

  3387.           "type": "graph",

  3388.           "xaxis": {

  3389.             "mode": "time",

  3390.             "show": true,

  3391.             "values": []

  3392.           },

  3393.           "yaxes": [

  3394.             {

  3395.               "format": "s",

  3396.               "logBase": 1,

  3397.               "show": true

  3398.             },

  3399.             {

  3400.               "format": "short",

  3401.               "logBase": 1,

  3402.               "show": false

  3403.             }

  3404.           ],

  3405.           "yaxis": {

  3406.             "align": false

  3407.           }

  3408.         },

  3409.         {

  3410.           "aliasColors": {},

  3411.           "bars": false,

  3412.           "dashLength": 10,

  3413.           "dashes": false,

  3414.           "datasource": {

  3415.             "uid": "${DS_PROMETHEUS}"

  3416.           },

  3417.           "fieldConfig": {

  3418.             "defaults": {

  3419.               "links": []

  3420.             },

  3421.             "overrides": []

  3422.           },

  3423.           "fill": 1,

  3424.           "fillGradient": 0,

  3425.           "gridPos": {

  3426.             "h": 9,

  3427.             "w": 12,

  3428.             "x": 0,

  3429.             "y": 53

  3430.           },

  3431.           "hiddenSeries": false,

  3432.           "id": 103,

  3433.           "legend": {

  3434.             "avg": false,

  3435.             "current": false,

  3436.             "max": false,

  3437.             "min": false,

  3438.             "show": true,

  3439.             "total": false,

  3440.             "values": false

  3441.           },

  3442.           "lines": true,

  3443.           "linewidth": 1,

  3444.           "links": [],

  3445.           "nullPointMode": "null",

  3446.           "options": {

  3447.             "alertThreshold": true

  3448.           },

  3449.           "percentage": false,

  3450.           "pluginVersion": "9.2.2",

  3451.           "pointradius": 5,

  3452.           "points": false,

  3453.           "renderer": "flot",

  3454.           "seriesOverrides": [

  3455.             {

  3456.               "alias": "Total",

  3457.               "color": "rgb(255, 255, 255)",

  3458.               "fill": 0,

  3459.               "linewidth": 3

  3460.             }

  3461.           ],

  3462.           "spaceLength": 10,

  3463.           "stack": false,

  3464.           "steppedLine": false,

  3465.           "targets": [

  3466.             {

  3467.               "datasource": {

  3468.                 "uid": "${DS_PROMETHEUS}"

  3469.               },

  3470.               "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",

  3471.               "format": "time_series",

  3472.               "interval": "",

  3473.               "intervalFactor": 1,

  3474.               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",

  3475.               "refId": "A"

  3476.             },

  3477.             {

  3478.               "datasource": {

  3479.                 "uid": "${DS_PROMETHEUS}"

  3480.               },

  3481.               "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",

  3482.               "interval": "",

  3483.               "legendFormat": "Total",

  3484.               "refId": "B"

  3485.             }

  3486.           ],

  3487.           "thresholds": [],

  3488.           "timeRegions": [],

  3489.           "title": "Requests in flight",

  3490.           "tooltip": {

  3491.             "shared": false,

  3492.             "sort": 2,

  3493.             "value_type": "individual"

  3494.           },

  3495.           "type": "graph",

  3496.           "xaxis": {

  3497.             "mode": "time",

  3498.             "show": true,

  3499.             "values": []

  3500.           },

  3501.           "yaxes": [

  3502.             {

  3503.               "format": "short",

  3504.               "logBase": 1,

  3505.               "show": true

  3506.             },

  3507.             {

  3508.               "format": "short",

  3509.               "logBase": 1,

  3510.               "show": true

  3511.             }

  3512.           ],

  3513.           "yaxis": {

  3514.             "align": false

  3515.           }

  3516.         }

  3517.       ],

  3518.       "targets": [

  3519.         {

  3520.           "datasource": {

  3521.             "type": "prometheus",

  3522.             "uid": "${DS_PROMETHEUS}"

  3523.           },

  3524.           "refId": "A"

  3525.         }

  3526.       ],

  3527.       "title": "Requests",

  3528.       "type": "row"

  3529.     },

  3530.     {

  3531.       "collapsed": true,

  3532.       "datasource": {

  3533.         "type": "prometheus",

  3534.         "uid": "${DS_PROMETHEUS}"

  3535.       },

  3536.       "gridPos": {

  3537.         "h": 1,

  3538.         "w": 24,

  3539.         "x": 0,

  3540.         "y": 29

  3541.       },

  3542.       "id": 97,

  3543.       "panels": [

  3544.         {

  3545.           "aliasColors": {},

  3546.           "bars": false,

  3547.           "dashLength": 10,

  3548.           "dashes": false,

  3549.           "datasource": {

  3550.             "uid": "${DS_PROMETHEUS}"

  3551.           },

  3552.           "fieldConfig": {

  3553.             "defaults": {

  3554.               "links": []

  3555.             },

  3556.             "overrides": []

  3557.           },

  3558.           "fill": 1,

  3559.           "fillGradient": 0,

  3560.           "gridPos": {

  3561.             "h": 9,

  3562.             "w": 12,

  3563.             "x": 0,

  3564.             "y": 30

  3565.           },

  3566.           "hiddenSeries": false,

  3567.           "id": 99,

  3568.           "legend": {

  3569.             "avg": false,

  3570.             "current": false,

  3571.             "max": false,

  3572.             "min": false,

  3573.             "show": true,

  3574.             "total": false,

  3575.             "values": false

  3576.           },

  3577.           "lines": true,

  3578.           "linewidth": 1,

  3579.           "links": [],

  3580.           "nullPointMode": "null",

  3581.           "options": {

  3582.             "alertThreshold": true

  3583.           },

  3584.           "paceLength": 10,

  3585.           "percentage": false,

  3586.           "pluginVersion": "9.2.2",

  3587.           "pointradius": 5,

  3588.           "points": false,

  3589.           "renderer": "flot",

  3590.           "seriesOverrides": [],

  3591.           "spaceLength": 10,

  3592.           "stack": false,

  3593.           "steppedLine": false,

  3594.           "targets": [

  3595.             {

  3596.               "datasource": {

  3597.                 "uid": "${DS_PROMETHEUS}"

  3598.               },

  3599.               "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  3600.               "format": "time_series",

  3601.               "interval": "",

  3602.               "intervalFactor": 1,

  3603.               "legendFormat": "{{job}}-{{index}} {{name}}",

  3604.               "refId": "A"

  3605.             }

  3606.           ],

  3607.           "thresholds": [],

  3608.           "timeRegions": [],

  3609.           "title": "CPU usage by background jobs",

  3610.           "tooltip": {

  3611.             "shared": false,

  3612.             "sort": 0,

  3613.             "value_type": "individual"

  3614.           },

  3615.           "type": "graph",

  3616.           "xaxis": {

  3617.             "mode": "time",

  3618.             "show": true,

  3619.             "values": []

  3620.           },

  3621.           "yaxes": [

  3622.             {

  3623.               "format": "percentunit",

  3624.               "logBase": 1,

  3625.               "show": true

  3626.             },

  3627.             {

  3628.               "format": "short",

  3629.               "logBase": 1,

  3630.               "show": true

  3631.             }

  3632.           ],

  3633.           "yaxis": {

  3634.             "align": false

  3635.           }

  3636.         },

  3637.         {

  3638.           "aliasColors": {},

  3639.           "bars": false,

  3640.           "dashLength": 10,

  3641.           "dashes": false,

  3642.           "datasource": {

  3643.             "uid": "${DS_PROMETHEUS}"

  3644.           },

  3645.           "fieldConfig": {

  3646.             "defaults": {

  3647.               "links": []

  3648.             },

  3649.             "overrides": []

  3650.           },

  3651.           "fill": 1,

  3652.           "fillGradient": 0,

  3653.           "gridPos": {

  3654.             "h": 9,

  3655.             "w": 12,

  3656.             "x": 12,

  3657.             "y": 30

  3658.           },

  3659.           "hiddenSeries": false,

  3660.           "id": 101,

  3661.           "legend": {

  3662.             "avg": false,

  3663.             "current": false,

  3664.             "max": false,

  3665.             "min": false,

  3666.             "show": true,

  3667.             "total": false,

  3668.             "values": false

  3669.           },

  3670.           "lines": true,

  3671.           "linewidth": 1,

  3672.           "links": [],

  3673.           "nullPointMode": "null",

  3674.           "options": {

  3675.             "alertThreshold": true

  3676.           },

  3677.           "paceLength": 10,

  3678.           "percentage": false,

  3679.           "pluginVersion": "9.2.2",

  3680.           "pointradius": 5,

  3681.           "points": false,

  3682.           "renderer": "flot",

  3683.           "seriesOverrides": [],

  3684.           "spaceLength": 10,

  3685.           "stack": false,

  3686.           "steppedLine": false,

  3687.           "targets": [

  3688.             {

  3689.               "datasource": {

  3690.                 "uid": "${DS_PROMETHEUS}"

  3691.               },

  3692.               "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) +  rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  3693.               "format": "time_series",

  3694.               "hide": false,

  3695.               "intervalFactor": 1,

  3696.               "legendFormat": "{{job}}-{{index}} {{name}}",

  3697.               "refId": "A"

  3698.             }

  3699.           ],

  3700.           "thresholds": [],

  3701.           "timeRegions": [],

  3702.           "title": "DB usage by background jobs (including scheduling time)",

  3703.           "tooltip": {

  3704.             "shared": false,

  3705.             "sort": 0,

  3706.             "value_type": "individual"

  3707.           },

  3708.           "type": "graph",

  3709.           "xaxis": {

  3710.             "mode": "time",

  3711.             "show": true,

  3712.             "values": []

  3713.           },

  3714.           "yaxes": [

  3715.             {

  3716.               "format": "percentunit",

  3717.               "logBase": 1,

  3718.               "show": true

  3719.             },

  3720.             {

  3721.               "format": "short",

  3722.               "logBase": 1,

  3723.               "show": true

  3724.             }

  3725.           ],

  3726.           "yaxis": {

  3727.             "align": false

  3728.           }

  3729.         },

  3730.         {

  3731.           "aliasColors": {},

  3732.           "bars": false,

  3733.           "dashLength": 10,

  3734.           "dashes": false,

  3735.           "datasource": {

  3736.             "uid": "${DS_PROMETHEUS}"

  3737.           },

  3738.           "fieldConfig": {

  3739.             "defaults": {

  3740.               "links": []

  3741.             },

  3742.             "overrides": []

  3743.           },

  3744.           "fill": 1,

  3745.           "fillGradient": 0,

  3746.           "gridPos": {

  3747.             "h": 8,

  3748.             "w": 12,

  3749.             "x": 0,

  3750.             "y": 39

  3751.           },

  3752.           "hiddenSeries": false,

  3753.           "id": 138,

  3754.           "legend": {

  3755.             "avg": false,

  3756.             "current": false,

  3757.             "max": false,

  3758.             "min": false,

  3759.             "show": true,

  3760.             "total": false,

  3761.             "values": false

  3762.           },

  3763.           "lines": true,

  3764.           "linewidth": 1,

  3765.           "nullPointMode": "null",

  3766.           "options": {

  3767.             "alertThreshold": true

  3768.           },

  3769.           "percentage": false,

  3770.           "pluginVersion": "9.2.2",

  3771.           "pointradius": 2,

  3772.           "points": false,

  3773.           "renderer": "flot",

  3774.           "seriesOverrides": [],

  3775.           "spaceLength": 10,

  3776.           "stack": false,

  3777.           "steppedLine": false,

  3778.           "targets": [

  3779.             {

  3780.               "datasource": {

  3781.                 "uid": "${DS_PROMETHEUS}"

  3782.               },

  3783.               "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",

  3784.               "legendFormat": "{{job}}-{{index}} {{name}}",

  3785.               "refId": "A"

  3786.             }

  3787.           ],

  3788.           "thresholds": [],

  3789.           "timeRegions": [],

  3790.           "title": "Background jobs in flight",

  3791.           "tooltip": {

  3792.             "shared": false,

  3793.             "sort": 0,

  3794.             "value_type": "individual"

  3795.           },

  3796.           "type": "graph",

  3797.           "xaxis": {

  3798.             "mode": "time",

  3799.             "show": true,

  3800.             "values": []

  3801.           },

  3802.           "yaxes": [

  3803.             {

  3804.               "format": "short",

  3805.               "logBase": 1,

  3806.               "show": true

  3807.             },

  3808.             {

  3809.               "format": "short",

  3810.               "logBase": 1,

  3811.               "show": true

  3812.             }

  3813.           ],

  3814.           "yaxis": {

  3815.             "align": false

  3816.           }

  3817.         }

  3818.       ],

  3819.       "targets": [

  3820.         {

  3821.           "datasource": {

  3822.             "type": "prometheus",

  3823.             "uid": "${DS_PROMETHEUS}"

  3824.           },

  3825.           "refId": "A"

  3826.         }

  3827.       ],

  3828.       "title": "Background jobs",

  3829.       "type": "row"

  3830.     },

  3831.     {

  3832.       "collapsed": true,

  3833.       "datasource": {

  3834.         "type": "prometheus",

  3835.         "uid": "${DS_PROMETHEUS}"

  3836.       },

  3837.       "gridPos": {

  3838.         "h": 1,

  3839.         "w": 24,

  3840.         "x": 0,

  3841.         "y": 30

  3842.       },

  3843.       "id": 81,

  3844.       "panels": [

  3845.         {

  3846.           "aliasColors": {},

  3847.           "bars": false,

  3848.           "dashLength": 10,

  3849.           "dashes": false,

  3850.           "datasource": {

  3851.             "uid": "${DS_PROMETHEUS}"

  3852.           },

  3853.           "fieldConfig": {

  3854.             "defaults": {

  3855.               "links": []

  3856.             },

  3857.             "overrides": []

  3858.           },

  3859.           "fill": 1,

  3860.           "fillGradient": 0,

  3861.           "gridPos": {

  3862.             "h": 9,

  3863.             "w": 12,

  3864.             "x": 0,

  3865.             "y": 31

  3866.           },

  3867.           "hiddenSeries": false,

  3868.           "id": 79,

  3869.           "legend": {

  3870.             "avg": false,

  3871.             "current": false,

  3872.             "max": false,

  3873.             "min": false,

  3874.             "show": true,

  3875.             "total": false,

  3876.             "values": false

  3877.           },

  3878.           "lines": true,

  3879.           "linewidth": 1,

  3880.           "links": [],

  3881.           "nullPointMode": "null",

  3882.           "options": {

  3883.             "alertThreshold": true

  3884.           },

  3885.           "paceLength": 10,

  3886.           "percentage": false,

  3887.           "pluginVersion": "9.2.2",

  3888.           "pointradius": 5,

  3889.           "points": false,

  3890.           "renderer": "flot",

  3891.           "seriesOverrides": [],

  3892.           "spaceLength": 10,

  3893.           "stack": false,

  3894.           "steppedLine": false,

  3895.           "targets": [

  3896.             {

  3897.               "datasource": {

  3898.                 "uid": "${DS_PROMETHEUS}"

  3899.               },

  3900.               "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",

  3901.               "format": "time_series",

  3902.               "intervalFactor": 1,

  3903.               "legendFormat": "successful txn rate",

  3904.               "refId": "A"

  3905.             },

  3906.             {

  3907.               "datasource": {

  3908.                 "uid": "${DS_PROMETHEUS}"

  3909.               },

  3910.               "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",

  3911.               "legendFormat": "failed txn rate",

  3912.               "refId": "B"

  3913.             }

  3914.           ],

  3915.           "thresholds": [],

  3916.           "timeRegions": [],

  3917.           "title": "Outgoing federation transaction rate",

  3918.           "tooltip": {

  3919.             "shared": true,

  3920.             "sort": 0,

  3921.             "value_type": "individual"

  3922.           },

  3923.           "type": "graph",

  3924.           "xaxis": {

  3925.             "mode": "time",

  3926.             "show": true,

  3927.             "values": []

  3928.           },

  3929.           "yaxes": [

  3930.             {

  3931.               "format": "hertz",

  3932.               "logBase": 1,

  3933.               "show": true

  3934.             },

  3935.             {

  3936.               "format": "short",

  3937.               "logBase": 1,

  3938.               "show": true

  3939.             }

  3940.           ],

  3941.           "yaxis": {

  3942.             "align": false

  3943.           }

  3944.         },

  3945.         {

  3946.           "aliasColors": {},

  3947.           "bars": false,

  3948.           "dashLength": 10,

  3949.           "dashes": false,

  3950.           "datasource": {

  3951.             "uid": "${DS_PROMETHEUS}"

  3952.           },

  3953.           "fieldConfig": {

  3954.             "defaults": {

  3955.               "links": []

  3956.             },

  3957.             "overrides": []

  3958.           },

  3959.           "fill": 1,

  3960.           "fillGradient": 0,

  3961.           "gridPos": {

  3962.             "h": 9,

  3963.             "w": 12,

  3964.             "x": 12,

  3965.             "y": 31

  3966.           },

  3967.           "hiddenSeries": false,

  3968.           "id": 83,

  3969.           "legend": {

  3970.             "avg": false,

  3971.             "current": false,

  3972.             "max": false,

  3973.             "min": false,

  3974.             "show": true,

  3975.             "total": false,

  3976.             "values": false

  3977.           },

  3978.           "lines": true,

  3979.           "linewidth": 1,

  3980.           "links": [],

  3981.           "nullPointMode": "null",

  3982.           "options": {

  3983.             "alertThreshold": true

  3984.           },

  3985.           "paceLength": 10,

  3986.           "percentage": false,

  3987.           "pluginVersion": "9.2.2",

  3988.           "pointradius": 5,

  3989.           "points": false,

  3990.           "renderer": "flot",

  3991.           "seriesOverrides": [],

  3992.           "spaceLength": 10,

  3993.           "stack": false,

  3994.           "steppedLine": false,

  3995.           "targets": [

  3996.             {

  3997.               "datasource": {

  3998.                 "uid": "${DS_PROMETHEUS}"

  3999.               },

  4000.               "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))",

  4001.               "format": "time_series",

  4002.               "intervalFactor": 1,

  4003.               "legendFormat": "pdus",

  4004.               "refId": "A"

  4005.             },

  4006.             {

  4007.               "datasource": {

  4008.                 "uid": "${DS_PROMETHEUS}"

  4009.               },

  4010.               "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))",

  4011.               "format": "time_series",

  4012.               "intervalFactor": 1,

  4013.               "legendFormat": "edus",

  4014.               "refId": "B"

  4015.             }

  4016.           ],

  4017.           "thresholds": [],

  4018.           "timeRegions": [],

  4019.           "title": "Incoming PDU/EDU rate",

  4020.           "tooltip": {

  4021.             "shared": true,

  4022.             "sort": 0,

  4023.             "value_type": "individual"

  4024.           },

  4025.           "type": "graph",

  4026.           "xaxis": {

  4027.             "mode": "time",

  4028.             "show": true,

  4029.             "values": []

  4030.           },

  4031.           "yaxes": [

  4032.             {

  4033.               "format": "hertz",

  4034.               "logBase": 1,

  4035.               "show": true

  4036.             },

  4037.             {

  4038.               "format": "short",

  4039.               "logBase": 1,

  4040.               "show": true

  4041.             }

  4042.           ],

  4043.           "yaxis": {

  4044.             "align": false

  4045.           }

  4046.         },

  4047.         {

  4048.           "aliasColors": {},

  4049.           "bars": false,

  4050.           "dashLength": 10,

  4051.           "dashes": false,

  4052.           "datasource": {

  4053.             "type": "prometheus",

  4054.             "uid": "${DS_PROMETHEUS}"

  4055.           },

  4056.           "fieldConfig": {

  4057.             "defaults": {

  4058.               "links": []

  4059.             },

  4060.             "overrides": []

  4061.           },

  4062.           "fill": 1,

  4063.           "fillGradient": 0,

  4064.           "gridPos": {

  4065.             "h": 9,

  4066.             "w": 12,

  4067.             "x": 0,

  4068.             "y": 40

  4069.           },

  4070.           "hiddenSeries": false,

  4071.           "id": 109,

  4072.           "legend": {

  4073.             "avg": false,

  4074.             "current": false,

  4075.             "max": false,

  4076.             "min": false,

  4077.             "show": true,

  4078.             "total": false,

  4079.             "values": false

  4080.           },

  4081.           "lines": true,

  4082.           "linewidth": 1,

  4083.           "links": [],

  4084.           "nullPointMode": "null",

  4085.           "options": {

  4086.             "alertThreshold": true

  4087.           },

  4088.           "paceLength": 10,

  4089.           "percentage": false,

  4090.           "pluginVersion": "9.2.2",

  4091.           "pointradius": 5,

  4092.           "points": false,

  4093.           "renderer": "flot",

  4094.           "seriesOverrides": [],

  4095.           "spaceLength": 10,

  4096.           "stack": false,

  4097.           "steppedLine": false,

  4098.           "targets": [

  4099.             {

  4100.               "datasource": {

  4101.                 "uid": "${DS_PROMETHEUS}"

  4102.               },

  4103.               "editorMode": "code",

  4104.               "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))",

  4105.               "format": "time_series",

  4106.               "interval": "",

  4107.               "intervalFactor": 1,

  4108.               "legendFormat": "pdus",

  4109.               "range": true,

  4110.               "refId": "A"

  4111.             },

  4112.             {

  4113.               "datasource": {

  4114.                 "uid": "${DS_PROMETHEUS}"

  4115.               },

  4116.               "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))",

  4117.               "format": "time_series",

  4118.               "intervalFactor": 1,

  4119.               "legendFormat": "edus",

  4120.               "refId": "B"

  4121.             }

  4122.           ],

  4123.           "thresholds": [],

  4124.           "timeRegions": [],

  4125.           "title": "Outgoing PDU/EDU rate",

  4126.           "tooltip": {

  4127.             "shared": true,

  4128.             "sort": 0,

  4129.             "value_type": "individual"

  4130.           },

  4131.           "type": "graph",

  4132.           "xaxis": {

  4133.             "mode": "time",

  4134.             "show": true,

  4135.             "values": []

  4136.           },

  4137.           "yaxes": [

  4138.             {

  4139.               "format": "hertz",

  4140.               "logBase": 1,

  4141.               "show": true

  4142.             },

  4143.             {

  4144.               "format": "short",

  4145.               "logBase": 1,

  4146.               "show": true

  4147.             }

  4148.           ],

  4149.           "yaxis": {

  4150.             "align": false

  4151.           }

  4152.         },

  4153.         {

  4154.           "aliasColors": {},

  4155.           "bars": false,

  4156.           "dashLength": 10,

  4157.           "dashes": false,

  4158.           "datasource": {

  4159.             "type": "prometheus",

  4160.             "uid": "${DS_PROMETHEUS}"

  4161.           },

  4162.           "fieldConfig": {

  4163.             "defaults": {

  4164.               "links": []

  4165.             },

  4166.             "overrides": []

  4167.           },

  4168.           "fill": 1,

  4169.           "fillGradient": 0,

  4170.           "gridPos": {

  4171.             "h": 9,

  4172.             "w": 12,

  4173.             "x": 12,

  4174.             "y": 40

  4175.           },

  4176.           "hiddenSeries": false,

  4177.           "id": 111,

  4178.           "legend": {

  4179.             "avg": false,

  4180.             "current": false,

  4181.             "max": false,

  4182.             "min": false,

  4183.             "show": true,

  4184.             "total": false,

  4185.             "values": false

  4186.           },

  4187.           "lines": true,

  4188.           "linewidth": 1,

  4189.           "links": [],

  4190.           "nullPointMode": "null",

  4191.           "options": {

  4192.             "alertThreshold": true

  4193.           },

  4194.           "paceLength": 10,

  4195.           "percentage": false,

  4196.           "pluginVersion": "9.2.2",

  4197.           "pointradius": 5,

  4198.           "points": false,

  4199.           "renderer": "flot",

  4200.           "seriesOverrides": [],

  4201.           "spaceLength": 10,

  4202.           "stack": false,

  4203.           "steppedLine": false,

  4204.           "targets": [

  4205.             {

  4206.               "datasource": {

  4207.                 "uid": "${DS_PROMETHEUS}"

  4208.               },

  4209.               "editorMode": "code",

  4210.               "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])",

  4211.               "format": "time_series",

  4212.               "interval": "",

  4213.               "intervalFactor": 1,

  4214.               "legendFormat": "{{type}}",

  4215.               "range": true,

  4216.               "refId": "A"

  4217.             }

  4218.           ],

  4219.           "thresholds": [],

  4220.           "timeRegions": [],

  4221.           "title": "Outgoing EDUs by type",

  4222.           "tooltip": {

  4223.             "shared": true,

  4224.             "sort": 0,

  4225.             "value_type": "individual"

  4226.           },

  4227.           "type": "graph",

  4228.           "xaxis": {

  4229.             "mode": "time",

  4230.             "show": true,

  4231.             "values": []

  4232.           },

  4233.           "yaxes": [

  4234.             {

  4235.               "$$hashKey": "object:462",

  4236.               "format": "hertz",

  4237.               "logBase": 1,

  4238.               "show": true

  4239.             },

  4240.             {

  4241.               "$$hashKey": "object:463",

  4242.               "format": "short",

  4243.               "logBase": 1,

  4244.               "show": true

  4245.             }

  4246.           ],

  4247.           "yaxis": {

  4248.             "align": false

  4249.           }

  4250.         },

  4251.         {

  4252.           "datasource": {

  4253.             "type": "prometheus",

  4254.             "uid": "${DS_PROMETHEUS}"

  4255.           },

  4256.           "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",

  4257.           "fieldConfig": {

  4258.             "defaults": {

  4259.               "color": {

  4260.                 "mode": "palette-classic"

  4261.               },

  4262.               "custom": {

  4263.                 "axisCenteredZero": false,

  4264.                 "axisColorMode": "text",

  4265.                 "axisLabel": "",

  4266.                 "axisPlacement": "auto",

  4267.                 "axisSoftMax": 60,

  4268.                 "axisSoftMin": 0,

  4269.                 "barAlignment": 0,

  4270.                 "drawStyle": "line",

  4271.                 "fillOpacity": 0,

  4272.                 "gradientMode": "none",

  4273.                 "hideFrom": {

  4274.                   "legend": false,

  4275.                   "tooltip": false,

  4276.                   "viz": false

  4277.                 },

  4278.                 "lineInterpolation": "linear",

  4279.                 "lineWidth": 1,

  4280.                 "pointSize": 5,

  4281.                 "scaleDistribution": {

  4282.                   "type": "linear"

  4283.                 },

  4284.                 "showPoints": "auto",

  4285.                 "spanNulls": false,

  4286.                 "stacking": {

  4287.                   "group": "A",

  4288.                   "mode": "none"

  4289.                 },

  4290.                 "thresholdsStyle": {

  4291.                   "mode": "line"

  4292.                 }

  4293.               },

  4294.               "mappings": [],

  4295.               "thresholds": {

  4296.                 "mode": "absolute",

  4297.                 "steps": [

  4298.                   {

  4299.                     "color": "green"

  4300.                   },

  4301.                   {

  4302.                     "color": "red",

  4303.                     "value": 60

  4304.                   }

  4305.                 ]

  4306.               },

  4307.               "unit": "m"

  4308.             },

  4309.             "overrides": [

  4310.               {

  4311.                 "__systemRef": "hideSeriesFrom",

  4312.                 "matcher": {

  4313.                   "id": "byNames",

  4314.                   "options": {

  4315.                     "mode": "exclude",

  4316.                     "names": [

  4317.                       "libera.chat "

  4318.                     ],

  4319.                     "prefix": "All except:",

  4320.                     "readOnly": true

  4321.                   }

  4322.                 },

  4323.                 "properties": [

  4324.                   {

  4325.                     "id": "custom.hideFrom",

  4326.                     "value": {

  4327.                       "legend": false,

  4328.                       "tooltip": false,

  4329.                       "viz": true

  4330.                     }

  4331.                   }

  4332.                 ]

  4333.               }

  4334.             ]

  4335.           },

  4336.           "gridPos": {

  4337.             "h": 8,

  4338.             "w": 12,

  4339.             "x": 0,

  4340.             "y": 49

  4341.           },

  4342.           "id": 243,

  4343.           "options": {

  4344.             "legend": {

  4345.               "calcs": [],

  4346.               "displayMode": "list",

  4347.               "placement": "bottom",

  4348.               "showLegend": true

  4349.             },

  4350.             "tooltip": {

  4351.               "mode": "single",

  4352.               "sort": "none"

  4353.             }

  4354.           },

  4355.           "targets": [

  4356.             {

  4357.               "datasource": {

  4358.                 "type": "prometheus",

  4359.                 "uid": "${DS_PROMETHEUS}"

  4360.               },

  4361.               "editorMode": "code",

  4362.               "exemplar": false,

  4363.               "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60",

  4364.               "instant": false,

  4365.               "legendFormat": "{{server_name}} ",

  4366.               "range": true,

  4367.               "refId": "A"

  4368.             }

  4369.           ],

  4370.           "title": "Age of last PDU received from nominated hosts",

  4371.           "type": "timeseries"

  4372.         },

  4373.         {

  4374.           "datasource": {

  4375.             "type": "prometheus",

  4376.             "uid": "${DS_PROMETHEUS}"

  4377.           },

  4378.           "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",

  4379.           "fieldConfig": {

  4380.             "defaults": {

  4381.               "color": {

  4382.                 "mode": "palette-classic"

  4383.               },

  4384.               "custom": {

  4385.                 "axisCenteredZero": false,

  4386.                 "axisColorMode": "text",

  4387.                 "axisLabel": "",

  4388.                 "axisPlacement": "auto",

  4389.                 "axisSoftMax": 60,

  4390.                 "axisSoftMin": 0,

  4391.                 "barAlignment": 0,

  4392.                 "drawStyle": "line",

  4393.                 "fillOpacity": 0,

  4394.                 "gradientMode": "none",

  4395.                 "hideFrom": {

  4396.                   "legend": false,

  4397.                   "tooltip": false,

  4398.                   "viz": false

  4399.                 },

  4400.                 "lineInterpolation": "linear",

  4401.                 "lineWidth": 1,

  4402.                 "pointSize": 5,

  4403.                 "scaleDistribution": {

  4404.                   "type": "linear"

  4405.                 },

  4406.                 "showPoints": "auto",

  4407.                 "spanNulls": false,

  4408.                 "stacking": {

  4409.                   "group": "A",

  4410.                   "mode": "none"

  4411.                 },

  4412.                 "thresholdsStyle": {

  4413.                   "mode": "line"

  4414.                 }

  4415.               },

  4416.               "mappings": [],

  4417.               "thresholds": {

  4418.                 "mode": "absolute",

  4419.                 "steps": [

  4420.                   {

  4421.                     "color": "green"

  4422.                   },

  4423.                   {

  4424.                     "color": "red",

  4425.                     "value": 60

  4426.                   }

  4427.                 ]

  4428.               },

  4429.               "unit": "m"

  4430.             },

  4431.             "overrides": [

  4432.               {

  4433.                 "__systemRef": "hideSeriesFrom",

  4434.                 "matcher": {

  4435.                   "id": "byNames",

  4436.                   "options": {

  4437.                     "mode": "exclude",

  4438.                     "names": [

  4439.                       "libera.chat"

  4440.                     ],

  4441.                     "prefix": "All except:",

  4442.                     "readOnly": true

  4443.                   }

  4444.                 },

  4445.                 "properties": [

  4446.                   {

  4447.                     "id": "custom.hideFrom",

  4448.                     "value": {

  4449.                       "legend": false,

  4450.                       "tooltip": false,

  4451.                       "viz": true

  4452.                     }

  4453.                   }

  4454.                 ]

  4455.               }

  4456.             ]

  4457.           },

  4458.           "gridPos": {

  4459.             "h": 8,

  4460.             "w": 12,

  4461.             "x": 12,

  4462.             "y": 49

  4463.           },

  4464.           "id": 241,

  4465.           "options": {

  4466.             "legend": {

  4467.               "calcs": [],

  4468.               "displayMode": "list",

  4469.               "placement": "bottom",

  4470.               "showLegend": true

  4471.             },

  4472.             "tooltip": {

  4473.               "mode": "single",

  4474.               "sort": "none"

  4475.             }

  4476.           },

  4477.           "targets": [

  4478.             {

  4479.               "datasource": {

  4480.                 "type": "prometheus",

  4481.                 "uid": "${DS_PROMETHEUS}"

  4482.               },

  4483.               "editorMode": "code",

  4484.               "exemplar": false,

  4485.               "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60",

  4486.               "instant": false,

  4487.               "legendFormat": "{{server_name}}",

  4488.               "range": true,

  4489.               "refId": "A"

  4490.             }

  4491.           ],

  4492.           "title": "Age of last PDU sent to nominated hosts",

  4493.           "type": "timeseries"

  4494.         },

  4495.         {

  4496.           "aliasColors": {},

  4497.           "bars": false,

  4498.           "dashLength": 10,

  4499.           "dashes": false,

  4500.           "datasource": {

  4501.             "type": "prometheus",

  4502.             "uid": "${DS_PROMETHEUS}"

  4503.           },

  4504.           "description": "The number of events in the in-memory queues ",

  4505.           "fieldConfig": {

  4506.             "defaults": {

  4507.               "links": []

  4508.             },

  4509.             "overrides": []

  4510.           },

  4511.           "fill": 1,

  4512.           "fillGradient": 0,

  4513.           "gridPos": {

  4514.             "h": 9,

  4515.             "w": 12,

  4516.             "x": 0,

  4517.             "y": 57

  4518.           },

  4519.           "hiddenSeries": false,

  4520.           "id": 142,

  4521.           "legend": {

  4522.             "avg": false,

  4523.             "current": false,

  4524.             "max": false,

  4525.             "min": false,

  4526.             "show": true,

  4527.             "total": false,

  4528.             "values": false

  4529.           },

  4530.           "lines": true,

  4531.           "linewidth": 1,

  4532.           "nullPointMode": "null",

  4533.           "options": {

  4534.             "alertThreshold": true

  4535.           },

  4536.           "percentage": false,

  4537.           "pluginVersion": "9.2.2",

  4538.           "pointradius": 2,

  4539.           "points": false,

  4540.           "renderer": "flot",

  4541.           "seriesOverrides": [],

  4542.           "spaceLength": 10,

  4543.           "stack": false,

  4544.           "steppedLine": false,

  4545.           "targets": [

  4546.             {

  4547.               "datasource": {

  4548.                 "type": "prometheus",

  4549.                 "uid": "${DS_PROMETHEUS}"

  4550.               },

  4551.               "editorMode": "code",

  4552.               "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4553.               "interval": "",

  4554.               "legendFormat": "pending PDUs {{job}}-{{index}}",

  4555.               "range": true,

  4556.               "refId": "A"

  4557.             },

  4558.             {

  4559.               "datasource": {

  4560.                 "type": "prometheus",

  4561.                 "uid": "${DS_PROMETHEUS}"

  4562.               },

  4563.               "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4564.               "interval": "",

  4565.               "legendFormat": "pending EDUs {{job}}-{{index}}",

  4566.               "refId": "B"

  4567.             }

  4568.           ],

  4569.           "thresholds": [],

  4570.           "timeRegions": [],

  4571.           "title": "In-memory federation transmission queues",

  4572.           "tooltip": {

  4573.             "shared": true,

  4574.             "sort": 0,

  4575.             "value_type": "individual"

  4576.           },

  4577.           "type": "graph",

  4578.           "xaxis": {

  4579.             "mode": "time",

  4580.             "show": true,

  4581.             "values": []

  4582.           },

  4583.           "yaxes": [

  4584.             {

  4585.               "$$hashKey": "object:547",

  4586.               "format": "short",

  4587.               "label": "events",

  4588.               "logBase": 1,

  4589.               "min": "0",

  4590.               "show": true

  4591.             },

  4592.             {

  4593.               "$$hashKey": "object:548",

  4594.               "format": "short",

  4595.               "label": "",

  4596.               "logBase": 1,

  4597.               "show": true

  4598.             }

  4599.           ],

  4600.           "yaxis": {

  4601.             "align": false

  4602.           }

  4603.         },

  4604.         {

  4605.           "aliasColors": {},

  4606.           "bars": false,

  4607.           "dashLength": 10,

  4608.           "dashes": false,

  4609.           "datasource": {

  4610.             "uid": "${DS_PROMETHEUS}"

  4611.           },

  4612.           "description": "Number of events queued up on the master process for processing by the federation sender",

  4613.           "fieldConfig": {

  4614.             "defaults": {

  4615.               "links": []

  4616.             },

  4617.             "overrides": []

  4618.           },

  4619.           "fill": 1,

  4620.           "fillGradient": 0,

  4621.           "gridPos": {

  4622.             "h": 9,

  4623.             "w": 12,

  4624.             "x": 12,

  4625.             "y": 57

  4626.           },

  4627.           "hiddenSeries": false,

  4628.           "id": 140,

  4629.           "legend": {

  4630.             "avg": false,

  4631.             "current": false,

  4632.             "max": false,

  4633.             "min": false,

  4634.             "show": true,

  4635.             "total": false,

  4636.             "values": false

  4637.           },

  4638.           "lines": true,

  4639.           "linewidth": 1,

  4640.           "links": [],

  4641.           "nullPointMode": "null",

  4642.           "options": {

  4643.             "alertThreshold": true

  4644.           },

  4645.           "paceLength": 10,

  4646.           "percentage": false,

  4647.           "pluginVersion": "9.2.2",

  4648.           "pointradius": 5,

  4649.           "points": false,

  4650.           "renderer": "flot",

  4651.           "seriesOverrides": [],

  4652.           "spaceLength": 10,

  4653.           "stack": false,

  4654.           "steppedLine": false,

  4655.           "targets": [

  4656.             {

  4657.               "datasource": {

  4658.                 "uid": "${DS_PROMETHEUS}"

  4659.               },

  4660.               "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4661.               "format": "time_series",

  4662.               "interval": "",

  4663.               "intervalFactor": 1,

  4664.               "legendFormat": "presence changed",

  4665.               "refId": "A"

  4666.             },

  4667.             {

  4668.               "datasource": {

  4669.                 "uid": "${DS_PROMETHEUS}"

  4670.               },

  4671.               "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4672.               "format": "time_series",

  4673.               "hide": false,

  4674.               "interval": "",

  4675.               "intervalFactor": 1,

  4676.               "legendFormat": "presence map",

  4677.               "refId": "B"

  4678.             },

  4679.             {

  4680.               "datasource": {

  4681.                 "uid": "${DS_PROMETHEUS}"

  4682.               },

  4683.               "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4684.               "format": "time_series",

  4685.               "hide": false,

  4686.               "interval": "",

  4687.               "intervalFactor": 1,

  4688.               "legendFormat": "presence destinations",

  4689.               "refId": "E"

  4690.             },

  4691.             {

  4692.               "datasource": {

  4693.                 "uid": "${DS_PROMETHEUS}"

  4694.               },

  4695.               "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4696.               "format": "time_series",

  4697.               "hide": false,

  4698.               "interval": "",

  4699.               "intervalFactor": 1,

  4700.               "legendFormat": "keyed edus",

  4701.               "refId": "C"

  4702.             },

  4703.             {

  4704.               "datasource": {

  4705.                 "uid": "${DS_PROMETHEUS}"

  4706.               },

  4707.               "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4708.               "format": "time_series",

  4709.               "hide": false,

  4710.               "interval": "",

  4711.               "intervalFactor": 1,

  4712.               "legendFormat": "other edus",

  4713.               "refId": "D"

  4714.             },

  4715.             {

  4716.               "datasource": {

  4717.                 "uid": "${DS_PROMETHEUS}"

  4718.               },

  4719.               "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  4720.               "format": "time_series",

  4721.               "hide": false,

  4722.               "interval": "",

  4723.               "intervalFactor": 1,

  4724.               "legendFormat": "stream positions",

  4725.               "refId": "F"

  4726.             }

  4727.           ],

  4728.           "thresholds": [],

  4729.           "timeRegions": [],

  4730.           "title": "Outgoing EDU queues on master",

  4731.           "tooltip": {

  4732.             "shared": true,

  4733.             "sort": 0,

  4734.             "value_type": "individual"

  4735.           },

  4736.           "type": "graph",

  4737.           "xaxis": {

  4738.             "mode": "time",

  4739.             "show": true,

  4740.             "values": []

  4741.           },

  4742.           "yaxes": [

  4743.             {

  4744.               "format": "none",

  4745.               "logBase": 1,

  4746.               "min": "0",

  4747.               "show": true

  4748.             },

  4749.             {

  4750.               "format": "short",

  4751.               "logBase": 1,

  4752.               "show": true

  4753.             }

  4754.           ],

  4755.           "yaxis": {

  4756.             "align": false

  4757.           }

  4758.         },

  4759.         {

  4760.           "cards": {

  4761.             "cardPadding": -1

  4762.           },

  4763.           "color": {

  4764.             "cardColor": "#b4ff00",

  4765.             "colorScale": "sqrt",

  4766.             "colorScheme": "interpolateInferno",

  4767.             "exponent": 0.5,

  4768.             "min": 0,

  4769.             "mode": "spectrum"

  4770.           },

  4771.           "dataFormat": "tsbuckets",

  4772.           "datasource": {

  4773.             "uid": "${DS_PROMETHEUS}"

  4774.           },

  4775.           "fieldConfig": {

  4776.             "defaults": {

  4777.               "custom": {

  4778.                 "hideFrom": {

  4779.                   "legend": false,

  4780.                   "tooltip": false,

  4781.                   "viz": false

  4782.                 },

  4783.                 "scaleDistribution": {

  4784.                   "type": "linear"

  4785.                 }

  4786.               }

  4787.             },

  4788.             "overrides": []

  4789.           },

  4790.           "gridPos": {

  4791.             "h": 9,

  4792.             "w": 12,

  4793.             "x": 0,

  4794.             "y": 66

  4795.           },

  4796.           "heatmap": {},

  4797.           "hideZeroBuckets": false,

  4798.           "highlightCards": true,

  4799.           "id": 166,

  4800.           "legend": {

  4801.             "show": false

  4802.           },

  4803.           "links": [],

  4804.           "options": {

  4805.             "calculate": false,

  4806.             "calculation": {},

  4807.             "cellGap": -1,

  4808.             "cellValues": {

  4809.               "decimals": 2

  4810.             },

  4811.             "color": {

  4812.               "exponent": 0.5,

  4813.               "fill": "#b4ff00",

  4814.               "min": 0,

  4815.               "mode": "scheme",

  4816.               "reverse": false,

  4817.               "scale": "exponential",

  4818.               "scheme": "Inferno",

  4819.               "steps": 128

  4820.             },

  4821.             "exemplars": {

  4822.               "color": "rgba(255,0,255,0.7)"

  4823.             },

  4824.             "filterValues": {

  4825.               "le": 1e-9

  4826.             },

  4827.             "legend": {

  4828.               "show": false

  4829.             },

  4830.             "rowsFrame": {

  4831.               "layout": "auto"

  4832.             },

  4833.             "showValue": "never",

  4834.             "tooltip": {

  4835.               "show": true,

  4836.               "yHistogram": true

  4837.             },

  4838.             "yAxis": {

  4839.               "axisPlacement": "left",

  4840.               "decimals": 0,

  4841.               "reverse": false,

  4842.               "unit": "s"

  4843.             }

  4844.           },

  4845.           "pluginVersion": "9.2.2",

  4846.           "reverseYBuckets": false,

  4847.           "targets": [

  4848.             {

  4849.               "datasource": {

  4850.                 "uid": "${DS_PROMETHEUS}"

  4851.               },

  4852.               "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",

  4853.               "format": "heatmap",

  4854.               "instant": false,

  4855.               "interval": "",

  4856.               "intervalFactor": 1,

  4857.               "legendFormat": "{{ le }}",

  4858.               "refId": "A"

  4859.             }

  4860.           ],

  4861.           "title": "Federation send PDU lag",

  4862.           "tooltip": {

  4863.             "show": true,

  4864.             "showHistogram": true

  4865.           },

  4866.           "tooltipDecimals": 2,

  4867.           "type": "heatmap",

  4868.           "xAxis": {

  4869.             "show": true

  4870.           },

  4871.           "yAxis": {

  4872.             "decimals": 0,

  4873.             "format": "s",

  4874.             "logBase": 1,

  4875.             "show": true

  4876.           },

  4877.           "yBucketBound": "auto"

  4878.         },

  4879.         {

  4880.           "aliasColors": {},

  4881.           "bars": false,

  4882.           "dashLength": 10,

  4883.           "dashes": false,

  4884.           "datasource": {

  4885.             "uid": "${DS_PROMETHEUS}"

  4886.           },

  4887.           "fieldConfig": {

  4888.             "defaults": {

  4889.               "links": []

  4890.             },

  4891.             "overrides": []

  4892.           },

  4893.           "fill": 0,

  4894.           "fillGradient": 0,

  4895.           "gridPos": {

  4896.             "h": 9,

  4897.             "w": 12,

  4898.             "x": 12,

  4899.             "y": 66

  4900.           },

  4901.           "hiddenSeries": false,

  4902.           "id": 162,

  4903.           "legend": {

  4904.             "avg": false,

  4905.             "current": false,

  4906.             "max": false,

  4907.             "min": false,

  4908.             "rightSide": false,

  4909.             "show": true,

  4910.             "total": false,

  4911.             "values": false

  4912.           },

  4913.           "lines": true,

  4914.           "linewidth": 0,

  4915.           "links": [],

  4916.           "nullPointMode": "connected",

  4917.           "options": {

  4918.             "alertThreshold": true

  4919.           },

  4920.           "paceLength": 10,

  4921.           "percentage": false,

  4922.           "pluginVersion": "9.2.2",

  4923.           "pointradius": 5,

  4924.           "points": false,

  4925.           "renderer": "flot",

  4926.           "seriesOverrides": [

  4927.             {

  4928.               "alias": "Avg",

  4929.               "fill": 0,

  4930.               "linewidth": 3

  4931.             },

  4932.             {

  4933.               "alias": "99%",

  4934.               "color": "#C4162A",

  4935.               "fillBelowTo": "90%"

  4936.             },

  4937.             {

  4938.               "alias": "90%",

  4939.               "color": "#FF7383",

  4940.               "fillBelowTo": "75%"

  4941.             },

  4942.             {

  4943.               "alias": "75%",

  4944.               "color": "#FFEE52",

  4945.               "fillBelowTo": "50%"

  4946.             },

  4947.             {

  4948.               "alias": "50%",

  4949.               "color": "#73BF69",

  4950.               "fillBelowTo": "25%"

  4951.             },

  4952.             {

  4953.               "alias": "25%",

  4954.               "color": "#1F60C4",

  4955.               "fillBelowTo": "5%"

  4956.             },

  4957.             {

  4958.               "alias": "5%",

  4959.               "lines": false

  4960.             },

  4961.             {

  4962.               "alias": "Average",

  4963.               "color": "rgb(255, 255, 255)",

  4964.               "lines": true,

  4965.               "linewidth": 3

  4966.             }

  4967.           ],

  4968.           "spaceLength": 10,

  4969.           "stack": false,

  4970.           "steppedLine": false,

  4971.           "targets": [

  4972.             {

  4973.               "datasource": {

  4974.                 "uid": "${DS_PROMETHEUS}"

  4975.               },

  4976.               "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  4977.               "format": "time_series",

  4978.               "interval": "",

  4979.               "intervalFactor": 1,

  4980.               "legendFormat": "99%",

  4981.               "refId": "D"

  4982.             },

  4983.             {

  4984.               "datasource": {

  4985.                 "uid": "${DS_PROMETHEUS}"

  4986.               },

  4987.               "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  4988.               "format": "time_series",

  4989.               "interval": "",

  4990.               "intervalFactor": 1,

  4991.               "legendFormat": "90%",

  4992.               "refId": "A"

  4993.             },

  4994.             {

  4995.               "datasource": {

  4996.                 "uid": "${DS_PROMETHEUS}"

  4997.               },

  4998.               "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  4999.               "format": "time_series",

  5000.               "interval": "",

  5001.               "intervalFactor": 1,

  5002.               "legendFormat": "75%",

  5003.               "refId": "C"

  5004.             },

  5005.             {

  5006.               "datasource": {

  5007.                 "uid": "${DS_PROMETHEUS}"

  5008.               },

  5009.               "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  5010.               "format": "time_series",

  5011.               "interval": "",

  5012.               "intervalFactor": 1,

  5013.               "legendFormat": "50%",

  5014.               "refId": "B"

  5015.             },

  5016.             {

  5017.               "datasource": {

  5018.                 "uid": "${DS_PROMETHEUS}"

  5019.               },

  5020.               "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  5021.               "interval": "",

  5022.               "legendFormat": "25%",

  5023.               "refId": "F"

  5024.             },

  5025.             {

  5026.               "datasource": {

  5027.                 "uid": "${DS_PROMETHEUS}"

  5028.               },

  5029.               "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  5030.               "interval": "",

  5031.               "legendFormat": "5%",

  5032.               "refId": "G"

  5033.             },

  5034.             {

  5035.               "datasource": {

  5036.                 "uid": "${DS_PROMETHEUS}"

  5037.               },

  5038.               "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",

  5039.               "interval": "",

  5040.               "legendFormat": "Average",

  5041.               "refId": "H"

  5042.             }

  5043.           ],

  5044.           "thresholds": [

  5045.             {

  5046.               "colorMode": "warning",

  5047.               "fill": false,

  5048.               "line": true,

  5049.               "op": "gt",

  5050.               "value": 0.25,

  5051.               "yaxis": "left"

  5052.             },

  5053.             {

  5054.               "colorMode": "critical",

  5055.               "fill": false,

  5056.               "line": true,

  5057.               "op": "gt",

  5058.               "value": 1,

  5059.               "yaxis": "left"

  5060.             }

  5061.           ],

  5062.           "timeRegions": [],

  5063.           "title": "Federation send PDU lag quantiles",

  5064.           "tooltip": {

  5065.             "shared": true,

  5066.             "sort": 2,

  5067.             "value_type": "individual"

  5068.           },

  5069.           "type": "graph",

  5070.           "xaxis": {

  5071.             "mode": "time",

  5072.             "show": true,

  5073.             "values": []

  5074.           },

  5075.           "yaxes": [

  5076.             {

  5077.               "format": "s",

  5078.               "label": "",

  5079.               "logBase": 1,

  5080.               "min": "0",

  5081.               "show": true

  5082.             },

  5083.             {

  5084.               "format": "hertz",

  5085.               "label": "",

  5086.               "logBase": 1,

  5087.               "min": "0",

  5088.               "show": true

  5089.             }

  5090.           ],

  5091.           "yaxis": {

  5092.             "align": false

  5093.           }

  5094.         },

  5095.         {

  5096.           "cards": {

  5097.             "cardPadding": -1

  5098.           },

  5099.           "color": {

  5100.             "cardColor": "#b4ff00",

  5101.             "colorScale": "sqrt",

  5102.             "colorScheme": "interpolateInferno",

  5103.             "exponent": 0.5,

  5104.             "min": 0,

  5105.             "mode": "spectrum"

  5106.           },

  5107.           "dataFormat": "tsbuckets",

  5108.           "datasource": {

  5109.             "uid": "${DS_PROMETHEUS}"

  5110.           },

  5111.           "fieldConfig": {

  5112.             "defaults": {

  5113.               "custom": {

  5114.                 "hideFrom": {

  5115.                   "legend": false,

  5116.                   "tooltip": false,

  5117.                   "viz": false

  5118.                 },

  5119.                 "scaleDistribution": {

  5120.                   "type": "linear"

  5121.                 }

  5122.               }

  5123.             },

  5124.             "overrides": []

  5125.           },

  5126.           "gridPos": {

  5127.             "h": 9,

  5128.             "w": 12,

  5129.             "x": 0,

  5130.             "y": 75

  5131.           },

  5132.           "heatmap": {},

  5133.           "hideZeroBuckets": false,

  5134.           "highlightCards": true,

  5135.           "id": 164,

  5136.           "legend": {

  5137.             "show": false

  5138.           },

  5139.           "links": [],

  5140.           "options": {

  5141.             "calculate": false,

  5142.             "calculation": {},

  5143.             "cellGap": -1,

  5144.             "cellValues": {

  5145.               "decimals": 2

  5146.             },

  5147.             "color": {

  5148.               "exponent": 0.5,

  5149.               "fill": "#b4ff00",

  5150.               "min": 0,

  5151.               "mode": "scheme",

  5152.               "reverse": false,

  5153.               "scale": "exponential",

  5154.               "scheme": "Inferno",

  5155.               "steps": 128

  5156.             },

  5157.             "exemplars": {

  5158.               "color": "rgba(255,0,255,0.7)"

  5159.             },

  5160.             "filterValues": {

  5161.               "le": 1e-9

  5162.             },

  5163.             "legend": {

  5164.               "show": false

  5165.             },

  5166.             "rowsFrame": {

  5167.               "layout": "auto"

  5168.             },

  5169.             "showValue": "never",

  5170.             "tooltip": {

  5171.               "show": true,

  5172.               "yHistogram": true

  5173.             },

  5174.             "yAxis": {

  5175.               "axisPlacement": "left",

  5176.               "decimals": 0,

  5177.               "reverse": false,

  5178.               "unit": "s"

  5179.             }

  5180.           },

  5181.           "pluginVersion": "9.2.2",

  5182.           "reverseYBuckets": false,

  5183.           "targets": [

  5184.             {

  5185.               "datasource": {

  5186.                 "uid": "${DS_PROMETHEUS}"

  5187.               },

  5188.               "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",

  5189.               "format": "heatmap",

  5190.               "instant": false,

  5191.               "interval": "",

  5192.               "intervalFactor": 1,

  5193.               "legendFormat": "{{ le }}",

  5194.               "refId": "A"

  5195.             }

  5196.           ],

  5197.           "title": "Handle inbound PDU time",

  5198.           "tooltip": {

  5199.             "show": true,

  5200.             "showHistogram": true

  5201.           },

  5202.           "tooltipDecimals": 2,

  5203.           "type": "heatmap",

  5204.           "xAxis": {

  5205.             "show": true

  5206.           },

  5207.           "yAxis": {

  5208.             "decimals": 0,

  5209.             "format": "s",

  5210.             "logBase": 1,

  5211.             "show": true

  5212.           },

  5213.           "yBucketBound": "auto"

  5214.         },

  5215.         {

  5216.           "aliasColors": {},

  5217.           "bars": false,

  5218.           "dashLength": 10,

  5219.           "dashes": false,

  5220.           "datasource": {

  5221.             "type": "prometheus",

  5222.             "uid": "${DS_PROMETHEUS}"

  5223.           },

  5224.           "editable": true,

  5225.           "error": false,

  5226.           "fieldConfig": {

  5227.             "defaults": {

  5228.               "links": []

  5229.             },

  5230.             "overrides": []

  5231.           },

  5232.           "fill": 1,

  5233.           "fillGradient": 0,

  5234.           "grid": {},

  5235.           "gridPos": {

  5236.             "h": 9,

  5237.             "w": 12,

  5238.             "x": 12,

  5239.             "y": 75

  5240.           },

  5241.           "hiddenSeries": false,

  5242.           "id": 203,

  5243.           "legend": {

  5244.             "avg": false,

  5245.             "current": false,

  5246.             "max": false,

  5247.             "min": false,

  5248.             "show": true,

  5249.             "total": false,

  5250.             "values": false

  5251.           },

  5252.           "lines": true,

  5253.           "linewidth": 2,

  5254.           "links": [],

  5255.           "nullPointMode": "null",

  5256.           "options": {

  5257.             "alertThreshold": true

  5258.           },

  5259.           "paceLength": 10,

  5260.           "percentage": false,

  5261.           "pluginVersion": "9.2.2",

  5262.           "pointradius": 5,

  5263.           "points": false,

  5264.           "renderer": "flot",

  5265.           "seriesOverrides": [],

  5266.           "spaceLength": 10,

  5267.           "stack": false,

  5268.           "steppedLine": false,

  5269.           "targets": [

  5270.             {

  5271.               "datasource": {

  5272.                 "uid": "${DS_PROMETHEUS}"

  5273.               },

  5274.               "editorMode": "code",

  5275.               "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",

  5276.               "format": "time_series",

  5277.               "interval": "",

  5278.               "intervalFactor": 1,

  5279.               "legendFormat": "{{job}}-{{index}}",

  5280.               "range": true,

  5281.               "refId": "A",

  5282.               "step": 4

  5283.             }

  5284.           ],

  5285.           "thresholds": [],

  5286.           "timeRegions": [],

  5287.           "title": "Age of oldest event in staging area",

  5288.           "tooltip": {

  5289.             "msResolution": false,

  5290.             "shared": true,

  5291.             "sort": 0,

  5292.             "value_type": "cumulative"

  5293.           },

  5294.           "type": "graph",

  5295.           "xaxis": {

  5296.             "mode": "time",

  5297.             "show": true,

  5298.             "values": []

  5299.           },

  5300.           "yaxes": [

  5301.             {

  5302.               "$$hashKey": "object:92",

  5303.               "format": "ms",

  5304.               "logBase": 1,

  5305.               "min": 0,

  5306.               "show": true

  5307.             },

  5308.             {

  5309.               "$$hashKey": "object:93",

  5310.               "format": "short",

  5311.               "logBase": 1,

  5312.               "show": true

  5313.             }

  5314.           ],

  5315.           "yaxis": {

  5316.             "align": false

  5317.           }

  5318.         },

  5319.         {

  5320.           "aliasColors": {},

  5321.           "bars": false,

  5322.           "dashLength": 10,

  5323.           "dashes": false,

  5324.           "datasource": {

  5325.             "type": "prometheus",

  5326.             "uid": "${DS_PROMETHEUS}"

  5327.           },

  5328.           "editable": true,

  5329.           "error": false,

  5330.           "fieldConfig": {

  5331.             "defaults": {

  5332.               "links": []

  5333.             },

  5334.             "overrides": []

  5335.           },

  5336.           "fill": 1,

  5337.           "fillGradient": 0,

  5338.           "grid": {},

  5339.           "gridPos": {

  5340.             "h": 9,

  5341.             "w": 12,

  5342.             "x": 0,

  5343.             "y": 84

  5344.           },

  5345.           "hiddenSeries": false,

  5346.           "id": 202,

  5347.           "legend": {

  5348.             "avg": false,

  5349.             "current": false,

  5350.             "max": false,

  5351.             "min": false,

  5352.             "show": true,

  5353.             "total": false,

  5354.             "values": false

  5355.           },

  5356.           "lines": true,

  5357.           "linewidth": 2,

  5358.           "links": [],

  5359.           "nullPointMode": "null",

  5360.           "options": {

  5361.             "alertThreshold": true

  5362.           },

  5363.           "paceLength": 10,

  5364.           "percentage": false,

  5365.           "pluginVersion": "9.2.2",

  5366.           "pointradius": 5,

  5367.           "points": false,

  5368.           "renderer": "flot",

  5369.           "seriesOverrides": [],

  5370.           "spaceLength": 10,

  5371.           "stack": false,

  5372.           "steppedLine": false,

  5373.           "targets": [

  5374.             {

  5375.               "datasource": {

  5376.                 "uid": "${DS_PROMETHEUS}"

  5377.               },

  5378.               "editorMode": "code",

  5379.               "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",

  5380.               "format": "time_series",

  5381.               "interval": "",

  5382.               "intervalFactor": 1,

  5383.               "legendFormat": "{{job}}-{{index}}",

  5384.               "range": true,

  5385.               "refId": "A",

  5386.               "step": 4

  5387.             }

  5388.           ],

  5389.           "thresholds": [],

  5390.           "timeRegions": [],

  5391.           "title": "Number of events in federation staging area",

  5392.           "tooltip": {

  5393.             "msResolution": false,

  5394.             "shared": true,

  5395.             "sort": 0,

  5396.             "value_type": "cumulative"

  5397.           },

  5398.           "type": "graph",

  5399.           "xaxis": {

  5400.             "mode": "time",

  5401.             "show": true,

  5402.             "values": []

  5403.           },

  5404.           "yaxes": [

  5405.             {

  5406.               "$$hashKey": "object:92",

  5407.               "format": "none",

  5408.               "logBase": 1,

  5409.               "min": 0,

  5410.               "show": true

  5411.             },

  5412.             {

  5413.               "$$hashKey": "object:93",

  5414.               "format": "short",

  5415.               "logBase": 1,

  5416.               "show": true

  5417.             }

  5418.           ],

  5419.           "yaxis": {

  5420.             "align": false

  5421.           }

  5422.         },

  5423.         {

  5424.           "aliasColors": {},

  5425.           "bars": false,

  5426.           "dashLength": 10,

  5427.           "dashes": false,

  5428.           "datasource": {

  5429.             "type": "prometheus",

  5430.             "uid": "${DS_PROMETHEUS}"

  5431.           },

  5432.           "fill": 1,

  5433.           "fillGradient": 0,

  5434.           "gridPos": {

  5435.             "h": 8,

  5436.             "w": 12,

  5437.             "x": 12,

  5438.             "y": 84

  5439.           },

  5440.           "hiddenSeries": false,

  5441.           "id": 205,

  5442.           "legend": {

  5443.             "avg": false,

  5444.             "current": false,

  5445.             "max": false,

  5446.             "min": false,

  5447.             "show": true,

  5448.             "total": false,

  5449.             "values": false

  5450.           },

  5451.           "lines": true,

  5452.           "linewidth": 1,

  5453.           "nullPointMode": "null",

  5454.           "options": {

  5455.             "alertThreshold": true

  5456.           },

  5457.           "percentage": false,

  5458.           "pluginVersion": "9.2.2",

  5459.           "pointradius": 2,

  5460.           "points": false,

  5461.           "renderer": "flot",

  5462.           "seriesOverrides": [],

  5463.           "spaceLength": 10,

  5464.           "stack": false,

  5465.           "steppedLine": false,

  5466.           "targets": [

  5467.             {

  5468.               "datasource": {

  5469.                 "type": "prometheus",

  5470.                 "uid": "${DS_PROMETHEUS}"

  5471.               },

  5472.               "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",

  5473.               "interval": "",

  5474.               "legendFormat": "soft-failed events",

  5475.               "refId": "A"

  5476.             }

  5477.           ],

  5478.           "thresholds": [],

  5479.           "timeRegions": [],

  5480.           "title": "Soft-failed event rate",

  5481.           "tooltip": {

  5482.             "shared": true,

  5483.             "sort": 0,

  5484.             "value_type": "individual"

  5485.           },

  5486.           "type": "graph",

  5487.           "xaxis": {

  5488.             "mode": "time",

  5489.             "show": true,

  5490.             "values": []

  5491.           },

  5492.           "yaxes": [

  5493.             {

  5494.               "$$hashKey": "object:131",

  5495.               "format": "hertz",

  5496.               "logBase": 1,

  5497.               "show": true

  5498.             },

  5499.             {

  5500.               "$$hashKey": "object:132",

  5501.               "format": "short",

  5502.               "logBase": 1,

  5503.               "show": false

  5504.             }

  5505.           ],

  5506.           "yaxis": {

  5507.             "align": false

  5508.           }

  5509.         }

  5510.       ],

  5511.       "targets": [

  5512.         {

  5513.           "datasource": {

  5514.             "type": "prometheus",

  5515.             "uid": "${DS_PROMETHEUS}"

  5516.           },

  5517.           "refId": "A"

  5518.         }

  5519.       ],

  5520.       "title": "Federation",

  5521.       "type": "row"

  5522.     },

  5523.     {

  5524.       "collapsed": true,

  5525.       "gridPos": {

  5526.         "h": 1,

  5527.         "w": 24,

  5528.         "x": 0,

  5529.         "y": 31

  5530.       },

  5531.       "id": 227,

  5532.       "panels": [

  5533.         {

  5534.           "datasource": {

  5535.             "type": "prometheus",

  5536.             "uid": "${DS_PROMETHEUS}"

  5537.           },

  5538.           "fieldConfig": {

  5539.             "defaults": {

  5540.               "color": {

  5541.                 "mode": "palette-classic"

  5542.               },

  5543.               "custom": {

  5544.                 "axisCenteredZero": false,

  5545.                 "axisColorMode": "text",

  5546.                 "axisLabel": "",

  5547.                 "axisPlacement": "auto",

  5548.                 "barAlignment": 0,

  5549.                 "drawStyle": "line",

  5550.                 "fillOpacity": 0,

  5551.                 "gradientMode": "none",

  5552.                 "hideFrom": {

  5553.                   "legend": false,

  5554.                   "tooltip": false,

  5555.                   "viz": false

  5556.                 },

  5557.                 "lineInterpolation": "linear",

  5558.                 "lineWidth": 1,

  5559.                 "pointSize": 5,

  5560.                 "scaleDistribution": {

  5561.                   "type": "linear"

  5562.                 },

  5563.                 "showPoints": "auto",

  5564.                 "spanNulls": false,

  5565.                 "stacking": {

  5566.                   "group": "A",

  5567.                   "mode": "none"

  5568.                 },

  5569.                 "thresholdsStyle": {

  5570.                   "mode": "off"

  5571.                 }

  5572.               },

  5573.               "mappings": [],

  5574.               "thresholds": {

  5575.                 "mode": "absolute",

  5576.                 "steps": [

  5577.                   {

  5578.                     "color": "green"

  5579.                   },

  5580.                   {

  5581.                     "color": "red",

  5582.                     "value": 80

  5583.                   }

  5584.                 ]

  5585.               }

  5586.             },

  5587.             "overrides": []

  5588.           },

  5589.           "gridPos": {

  5590.             "h": 8,

  5591.             "w": 12,

  5592.             "x": 0,

  5593.             "y": 154

  5594.           },

  5595.           "id": 239,

  5596.           "options": {

  5597.             "legend": {

  5598.               "calcs": [],

  5599.               "displayMode": "list",

  5600.               "placement": "bottom",

  5601.               "showLegend": true

  5602.             },

  5603.             "tooltip": {

  5604.               "mode": "single",

  5605.               "sort": "none"

  5606.             }

  5607.           },

  5608.           "targets": [

  5609.             {

  5610.               "datasource": {

  5611.                 "type": "prometheus",

  5612.                 "uid": "${DS_PROMETHEUS}"

  5613.               },

  5614.               "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))",

  5615.               "refId": "A"

  5616.             }

  5617.           ],

  5618.           "title": "Number of rate limit rejected requests",

  5619.           "type": "timeseries"

  5620.         },

  5621.         {

  5622.           "datasource": {

  5623.             "type": "prometheus",

  5624.             "uid": "${DS_PROMETHEUS}"

  5625.           },

  5626.           "fieldConfig": {

  5627.             "defaults": {

  5628.               "color": {

  5629.                 "mode": "palette-classic"

  5630.               },

  5631.               "custom": {

  5632.                 "axisCenteredZero": false,

  5633.                 "axisColorMode": "text",

  5634.                 "axisLabel": "",

  5635.                 "axisPlacement": "auto",

  5636.                 "barAlignment": 0,

  5637.                 "drawStyle": "line",

  5638.                 "fillOpacity": 0,

  5639.                 "gradientMode": "none",

  5640.                 "hideFrom": {

  5641.                   "legend": false,

  5642.                   "tooltip": false,

  5643.                   "viz": false

  5644.                 },

  5645.                 "lineInterpolation": "linear",

  5646.                 "lineWidth": 1,

  5647.                 "pointSize": 5,

  5648.                 "scaleDistribution": {

  5649.                   "type": "linear"

  5650.                 },

  5651.                 "showPoints": "auto",

  5652.                 "spanNulls": false,

  5653.                 "stacking": {

  5654.                   "group": "A",

  5655.                   "mode": "none"

  5656.                 },

  5657.                 "thresholdsStyle": {

  5658.                   "mode": "off"

  5659.                 }

  5660.               },

  5661.               "mappings": [],

  5662.               "thresholds": {

  5663.                 "mode": "absolute",

  5664.                 "steps": [

  5665.                   {

  5666.                     "color": "green"

  5667.                   },

  5668.                   {

  5669.                     "color": "red",

  5670.                     "value": 80

  5671.                   }

  5672.                 ]

  5673.               }

  5674.             },

  5675.             "overrides": []

  5676.           },

  5677.           "gridPos": {

  5678.             "h": 8,

  5679.             "w": 12,

  5680.             "x": 12,

  5681.             "y": 154

  5682.           },

  5683.           "id": 235,

  5684.           "options": {

  5685.             "legend": {

  5686.               "calcs": [],

  5687.               "displayMode": "list",

  5688.               "placement": "bottom",

  5689.               "showLegend": true

  5690.             },

  5691.             "tooltip": {

  5692.               "mode": "single",

  5693.               "sort": "none"

  5694.             }

  5695.           },

  5696.           "targets": [

  5697.             {

  5698.               "datasource": {

  5699.                 "type": "prometheus",

  5700.                 "uid": "${DS_PROMETHEUS}"

  5701.               },

  5702.               "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",

  5703.               "refId": "A"

  5704.             }

  5705.           ],

  5706.           "title": "Number of requests being slept by the rate limiter",

  5707.           "type": "timeseries"

  5708.         },

  5709.         {

  5710.           "datasource": {

  5711.             "type": "prometheus",

  5712.             "uid": "${DS_PROMETHEUS}"

  5713.           },

  5714.           "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322",

  5715.           "fieldConfig": {

  5716.             "defaults": {

  5717.               "color": {

  5718.                 "mode": "palette-classic"

  5719.               },

  5720.               "custom": {

  5721.                 "axisCenteredZero": false,

  5722.                 "axisColorMode": "text",

  5723.                 "axisLabel": "",

  5724.                 "axisPlacement": "auto",

  5725.                 "barAlignment": 0,

  5726.                 "drawStyle": "line",

  5727.                 "fillOpacity": 0,

  5728.                 "gradientMode": "none",

  5729.                 "hideFrom": {

  5730.                   "legend": false,

  5731.                   "tooltip": false,

  5732.                   "viz": false

  5733.                 },

  5734.                 "lineInterpolation": "linear",

  5735.                 "lineWidth": 1,

  5736.                 "pointSize": 5,

  5737.                 "scaleDistribution": {

  5738.                   "type": "linear"

  5739.                 },

  5740.                 "showPoints": "auto",

  5741.                 "spanNulls": false,

  5742.                 "stacking": {

  5743.                   "group": "A",

  5744.                   "mode": "none"

  5745.                 },

  5746.                 "thresholdsStyle": {

  5747.                   "mode": "off"

  5748.                 }

  5749.               },

  5750.               "mappings": [],

  5751.               "thresholds": {

  5752.                 "mode": "absolute",

  5753.                 "steps": [

  5754.                   {

  5755.                     "color": "green"

  5756.                   },

  5757.                   {

  5758.                     "color": "red",

  5759.                     "value": 80

  5760.                   }

  5761.                 ]

  5762.               }

  5763.             },

  5764.             "overrides": []

  5765.           },

  5766.           "gridPos": {

  5767.             "h": 8,

  5768.             "w": 12,

  5769.             "x": 0,

  5770.             "y": 162

  5771.           },

  5772.           "id": 237,

  5773.           "options": {

  5774.             "legend": {

  5775.               "calcs": [],

  5776.               "displayMode": "list",

  5777.               "placement": "bottom",

  5778.               "showLegend": true

  5779.             },

  5780.             "tooltip": {

  5781.               "mode": "single",

  5782.               "sort": "none"

  5783.             }

  5784.           },

  5785.           "pluginVersion": "9.0.4",

  5786.           "targets": [

  5787.             {

  5788.               "datasource": {

  5789.                 "type": "prometheus",

  5790.                 "uid": "${DS_PROMETHEUS}"

  5791.               },

  5792.               "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))",

  5793.               "refId": "A"

  5794.             }

  5795.           ],

  5796.           "title": "Number of hosts being rejected by the rate limiter",

  5797.           "type": "timeseries"

  5798.         },

  5799.         {

  5800.           "datasource": {

  5801.             "type": "prometheus",

  5802.             "uid": "${DS_PROMETHEUS}"

  5803.           },

  5804.           "description": "https://github.com/matrix-org/synapse/pull/13541",

  5805.           "fieldConfig": {

  5806.             "defaults": {

  5807.               "color": {

  5808.                 "mode": "palette-classic"

  5809.               },

  5810.               "custom": {

  5811.                 "axisCenteredZero": false,

  5812.                 "axisColorMode": "text",

  5813.                 "axisLabel": "",

  5814.                 "axisPlacement": "auto",

  5815.                 "barAlignment": 0,

  5816.                 "drawStyle": "line",

  5817.                 "fillOpacity": 0,

  5818.                 "gradientMode": "none",

  5819.                 "hideFrom": {

  5820.                   "legend": false,

  5821.                   "tooltip": false,

  5822.                   "viz": false

  5823.                 },

  5824.                 "lineInterpolation": "linear",

  5825.                 "lineWidth": 1,

  5826.                 "pointSize": 5,

  5827.                 "scaleDistribution": {

  5828.                   "type": "linear"

  5829.                 },

  5830.                 "showPoints": "auto",

  5831.                 "spanNulls": false,

  5832.                 "stacking": {

  5833.                   "group": "A",

  5834.                   "mode": "none"

  5835.                 },

  5836.                 "thresholdsStyle": {

  5837.                   "mode": "off"

  5838.                 }

  5839.               },

  5840.               "mappings": [],

  5841.               "thresholds": {

  5842.                 "mode": "absolute",

  5843.                 "steps": [

  5844.                   {

  5845.                     "color": "green"

  5846.                   },

  5847.                   {

  5848.                     "color": "red",

  5849.                     "value": 80

  5850.                   }

  5851.                 ]

  5852.               }

  5853.             },

  5854.             "overrides": []

  5855.           },

  5856.           "gridPos": {

  5857.             "h": 8,

  5858.             "w": 12,

  5859.             "x": 12,

  5860.             "y": 162

  5861.           },

  5862.           "id": 233,

  5863.           "options": {

  5864.             "legend": {

  5865.               "calcs": [],

  5866.               "displayMode": "list",

  5867.               "placement": "bottom",

  5868.               "showLegend": true

  5869.             },

  5870.             "tooltip": {

  5871.               "mode": "single",

  5872.               "sort": "none"

  5873.             }

  5874.           },

  5875.           "pluginVersion": "9.0.4",

  5876.           "targets": [

  5877.             {

  5878.               "datasource": {

  5879.                 "type": "prometheus",

  5880.                 "uid": "${DS_PROMETHEUS}"

  5881.               },

  5882.               "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))",

  5883.               "refId": "A"

  5884.             }

  5885.           ],

  5886.           "title": "Number of hosts being slept by the rate limiter",

  5887.           "type": "timeseries"

  5888.         },

  5889.         {

  5890.           "aliasColors": {},

  5891.           "bars": false,

  5892.           "dashLength": 10,

  5893.           "dashes": false,

  5894.           "datasource": {

  5895.             "type": "prometheus",

  5896.             "uid": "${DS_PROMETHEUS}"

  5897.           },

  5898.           "description": "",

  5899.           "fieldConfig": {

  5900.             "defaults": {

  5901.               "links": []

  5902.             },

  5903.             "overrides": []

  5904.           },

  5905.           "fill": 0,

  5906.           "fillGradient": 0,

  5907.           "gridPos": {

  5908.             "h": 8,

  5909.             "w": 12,

  5910.             "x": 0,

  5911.             "y": 170

  5912.           },

  5913.           "hiddenSeries": false,

  5914.           "id": 229,

  5915.           "legend": {

  5916.             "avg": false,

  5917.             "current": false,

  5918.             "max": false,

  5919.             "min": false,

  5920.             "rightSide": false,

  5921.             "show": true,

  5922.             "total": false,

  5923.             "values": false

  5924.           },

  5925.           "lines": true,

  5926.           "linewidth": 0,

  5927.           "links": [],

  5928.           "nullPointMode": "connected",

  5929.           "options": {

  5930.             "alertThreshold": true

  5931.           },

  5932.           "paceLength": 10,

  5933.           "percentage": false,

  5934.           "pluginVersion": "9.2.2",

  5935.           "pointradius": 5,

  5936.           "points": false,

  5937.           "renderer": "flot",

  5938.           "seriesOverrides": [

  5939.             {

  5940.               "$$hashKey": "object:276",

  5941.               "alias": "Avg",

  5942.               "fill": 0,

  5943.               "linewidth": 3

  5944.             },

  5945.             {

  5946.               "$$hashKey": "object:277",

  5947.               "alias": "99%",

  5948.               "color": "#C4162A",

  5949.               "fillBelowTo": "90%"

  5950.             },

  5951.             {

  5952.               "$$hashKey": "object:278",

  5953.               "alias": "90%",

  5954.               "color": "#FF7383",

  5955.               "fillBelowTo": "75%"

  5956.             },

  5957.             {

  5958.               "$$hashKey": "object:279",

  5959.               "alias": "75%",

  5960.               "color": "#FFEE52",

  5961.               "fillBelowTo": "50%"

  5962.             },

  5963.             {

  5964.               "$$hashKey": "object:280",

  5965.               "alias": "50%",

  5966.               "color": "#73BF69",

  5967.               "fillBelowTo": "25%"

  5968.             },

  5969.             {

  5970.               "$$hashKey": "object:281",

  5971.               "alias": "25%",

  5972.               "color": "#1F60C4",

  5973.               "fillBelowTo": "5%"

  5974.             },

  5975.             {

  5976.               "$$hashKey": "object:282",

  5977.               "alias": "5%",

  5978.               "lines": false

  5979.             },

  5980.             {

  5981.               "$$hashKey": "object:283",

  5982.               "alias": "Average",

  5983.               "color": "rgb(255, 255, 255)",

  5984.               "lines": true,

  5985.               "linewidth": 3

  5986.             },

  5987.             {

  5988.               "$$hashKey": "object:284",

  5989.               "alias": ">99%",

  5990.               "color": "#B877D9",

  5991.               "fill": 3,

  5992.               "lines": true

  5993.             }

  5994.           ],

  5995.           "spaceLength": 10,

  5996.           "stack": false,

  5997.           "steppedLine": false,

  5998.           "targets": [

  5999.             {

  6000.               "datasource": {

  6001.                 "uid": "${DS_PROMETHEUS}"

  6002.               },

  6003.               "editorMode": "code",

  6004.               "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6005.               "format": "time_series",

  6006.               "hide": false,

  6007.               "intervalFactor": 1,

  6008.               "legendFormat": ">99%",

  6009.               "range": true,

  6010.               "refId": "E"

  6011.             },

  6012.             {

  6013.               "datasource": {

  6014.                 "uid": "${DS_PROMETHEUS}"

  6015.               },

  6016.               "editorMode": "code",

  6017.               "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6018.               "format": "time_series",

  6019.               "intervalFactor": 1,

  6020.               "legendFormat": "99%",

  6021.               "range": true,

  6022.               "refId": "D"

  6023.             },

  6024.             {

  6025.               "datasource": {

  6026.                 "uid": "${DS_PROMETHEUS}"

  6027.               },

  6028.               "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6029.               "format": "time_series",

  6030.               "interval": "",

  6031.               "intervalFactor": 1,

  6032.               "legendFormat": "90%",

  6033.               "refId": "A"

  6034.             },

  6035.             {

  6036.               "datasource": {

  6037.                 "uid": "${DS_PROMETHEUS}"

  6038.               },

  6039.               "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6040.               "format": "time_series",

  6041.               "intervalFactor": 1,

  6042.               "legendFormat": "75%",

  6043.               "refId": "C"

  6044.             },

  6045.             {

  6046.               "datasource": {

  6047.                 "uid": "${DS_PROMETHEUS}"

  6048.               },

  6049.               "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6050.               "format": "time_series",

  6051.               "intervalFactor": 1,

  6052.               "legendFormat": "50%",

  6053.               "refId": "B"

  6054.             },

  6055.             {

  6056.               "datasource": {

  6057.                 "uid": "${DS_PROMETHEUS}"

  6058.               },

  6059.               "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6060.               "legendFormat": "25%",

  6061.               "refId": "F"

  6062.             },

  6063.             {

  6064.               "datasource": {

  6065.                 "uid": "${DS_PROMETHEUS}"

  6066.               },

  6067.               "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",

  6068.               "legendFormat": "5%",

  6069.               "refId": "G"

  6070.             },

  6071.             {

  6072.               "datasource": {

  6073.                 "uid": "${DS_PROMETHEUS}"

  6074.               },

  6075.               "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",

  6076.               "legendFormat": "Average",

  6077.               "refId": "H"

  6078.             }

  6079.           ],

  6080.           "thresholds": [

  6081.             {

  6082.               "$$hashKey": "object:283",

  6083.               "colorMode": "warning",

  6084.               "fill": false,

  6085.               "line": true,

  6086.               "op": "gt",

  6087.               "value": 1,

  6088.               "yaxis": "left"

  6089.             },

  6090.             {

  6091.               "$$hashKey": "object:284",

  6092.               "colorMode": "critical",

  6093.               "fill": false,

  6094.               "line": true,

  6095.               "op": "gt",

  6096.               "value": 2,

  6097.               "yaxis": "left"

  6098.             }

  6099.           ],

  6100.           "timeRegions": [],

  6101.           "title": "Rate limit queue wait time Quantiles (all workers)",

  6102.           "tooltip": {

  6103.             "shared": true,

  6104.             "sort": 2,

  6105.             "value_type": "individual"

  6106.           },

  6107.           "type": "graph",

  6108.           "xaxis": {

  6109.             "mode": "time",

  6110.             "show": true,

  6111.             "values": []

  6112.           },

  6113.           "yaxes": [

  6114.             {

  6115.               "$$hashKey": "object:255",

  6116.               "format": "s",

  6117.               "label": "",

  6118.               "logBase": 1,

  6119.               "min": "0",

  6120.               "show": true

  6121.             },

  6122.             {

  6123.               "$$hashKey": "object:256",

  6124.               "format": "hertz",

  6125.               "label": "",

  6126.               "logBase": 1,

  6127.               "min": "0",

  6128.               "show": true

  6129.             }

  6130.           ],

  6131.           "yaxis": {

  6132.             "align": false

  6133.           }

  6134.         },

  6135.         {

  6136.           "datasource": {

  6137.             "type": "prometheus",

  6138.             "uid": "${DS_PROMETHEUS}"

  6139.           },

  6140.           "fieldConfig": {

  6141.             "defaults": {

  6142.               "color": {

  6143.                 "mode": "palette-classic"

  6144.               },

  6145.               "custom": {

  6146.                 "axisCenteredZero": false,

  6147.                 "axisColorMode": "text",

  6148.                 "axisLabel": "",

  6149.                 "axisPlacement": "auto",

  6150.                 "barAlignment": 0,

  6151.                 "drawStyle": "line",

  6152.                 "fillOpacity": 0,

  6153.                 "gradientMode": "none",

  6154.                 "hideFrom": {

  6155.                   "legend": false,

  6156.                   "tooltip": false,

  6157.                   "viz": false

  6158.                 },

  6159.                 "lineInterpolation": "linear",

  6160.                 "lineWidth": 1,

  6161.                 "pointSize": 5,

  6162.                 "scaleDistribution": {

  6163.                   "type": "linear"

  6164.                 },

  6165.                 "showPoints": "auto",

  6166.                 "spanNulls": false,

  6167.                 "stacking": {

  6168.                   "group": "A",

  6169.                   "mode": "none"

  6170.                 },

  6171.                 "thresholdsStyle": {

  6172.                   "mode": "line"

  6173.                 }

  6174.               },

  6175.               "mappings": [],

  6176.               "thresholds": {

  6177.                 "mode": "absolute",

  6178.                 "steps": [

  6179.                   {

  6180.                     "color": "green"

  6181.                   }

  6182.                 ]

  6183.               },

  6184.               "unit": "hertz"

  6185.             },

  6186.             "overrides": [

  6187.               {

  6188.                 "matcher": {

  6189.                   "id": "byName",

  6190.                   "options": "Default reject threshold (50 requests within a second)"

  6191.                 },

  6192.                 "properties": [

  6193.                   {

  6194.                     "id": "color",

  6195.                     "value": {

  6196.                       "fixedColor": "red",

  6197.                       "mode": "fixed"

  6198.                     }

  6199.                   },

  6200.                   {

  6201.                     "id": "custom.lineWidth",

  6202.                     "value": 2

  6203.                   }

  6204.                 ]

  6205.               }

  6206.             ]

  6207.           },

  6208.           "gridPos": {

  6209.             "h": 8,

  6210.             "w": 12,

  6211.             "x": 12,

  6212.             "y": 170

  6213.           },

  6214.           "id": 231,

  6215.           "options": {

  6216.             "legend": {

  6217.               "calcs": [],

  6218.               "displayMode": "list",

  6219.               "placement": "bottom",

  6220.               "showLegend": true

  6221.             },

  6222.             "tooltip": {

  6223.               "mode": "single",

  6224.               "sort": "none"

  6225.             }

  6226.           },

  6227.           "targets": [

  6228.             {

  6229.               "datasource": {

  6230.                 "type": "prometheus",

  6231.                 "uid": "${DS_PROMETHEUS}"

  6232.               },

  6233.               "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",

  6234.               "refId": "A"

  6235.             },

  6236.             {

  6237.               "datasource": {

  6238.                 "type": "prometheus",

  6239.                 "uid": "${DS_PROMETHEUS}"

  6240.               },

  6241.               "editorMode": "code",

  6242.               "expr": "50",

  6243.               "hide": false,

  6244.               "legendFormat": "Default reject threshold (50 requests within a second)",

  6245.               "range": true,

  6246.               "refId": "B"

  6247.             }

  6248.           ],

  6249.           "title": "Rate of requests being slept by the rate limiter",

  6250.           "type": "timeseries"

  6251.         }

  6252.       ],

  6253.       "title": "Federation rate limiter",

  6254.       "type": "row"

  6255.     },

  6256.     {

  6257.       "collapsed": true,

  6258.       "datasource": {

  6259.         "type": "prometheus",

  6260.         "uid": "${DS_PROMETHEUS}"

  6261.       },

  6262.       "gridPos": {

  6263.         "h": 1,

  6264.         "w": 24,

  6265.         "x": 0,

  6266.         "y": 32

  6267.       },

  6268.       "id": 60,

  6269.       "panels": [

  6270.         {

  6271.           "datasource": {

  6272.             "type": "prometheus",

  6273.             "uid": "${DS_PROMETHEUS}"

  6274.           },

  6275.           "fieldConfig": {

  6276.             "defaults": {

  6277.               "color": {

  6278.                 "mode": "palette-classic"

  6279.               },

  6280.               "custom": {

  6281.                 "axisCenteredZero": false,

  6282.                 "axisColorMode": "text",

  6283.                 "axisLabel": "",

  6284.                 "axisPlacement": "auto",

  6285.                 "barAlignment": 0,

  6286.                 "drawStyle": "line",

  6287.                 "fillOpacity": 10,

  6288.                 "gradientMode": "none",

  6289.                 "hideFrom": {

  6290.                   "legend": false,

  6291.                   "tooltip": false,

  6292.                   "viz": false

  6293.                 },

  6294.                 "lineInterpolation": "linear",

  6295.                 "lineWidth": 1,

  6296.                 "pointSize": 5,

  6297.                 "scaleDistribution": {

  6298.                   "type": "linear"

  6299.                 },

  6300.                 "showPoints": "never",

  6301.                 "spanNulls": false,

  6302.                 "stacking": {

  6303.                   "group": "A",

  6304.                   "mode": "none"

  6305.                 },

  6306.                 "thresholdsStyle": {

  6307.                   "mode": "off"

  6308.                 }

  6309.               },

  6310.               "links": [],

  6311.               "mappings": [],

  6312.               "thresholds": {

  6313.                 "mode": "absolute",

  6314.                 "steps": [

  6315.                   {

  6316.                     "color": "green"

  6317.                   },

  6318.                   {

  6319.                     "color": "red",

  6320.                     "value": 80

  6321.                   }

  6322.                 ]

  6323.               },

  6324.               "unit": "hertz"

  6325.             },

  6326.             "overrides": []

  6327.           },

  6328.           "gridPos": {

  6329.             "h": 8,

  6330.             "w": 12,

  6331.             "x": 0,

  6332.             "y": 155

  6333.           },

  6334.           "id": 51,

  6335.           "links": [],

  6336.           "options": {

  6337.             "legend": {

  6338.               "calcs": [],

  6339.               "displayMode": "list",

  6340.               "placement": "bottom",

  6341.               "showLegend": true

  6342.             },

  6343.             "tooltip": {

  6344.               "mode": "multi",

  6345.               "sort": "none"

  6346.             }

  6347.           },

  6348.           "pluginVersion": "9.2.2",

  6349.           "targets": [

  6350.             {

  6351.               "datasource": {

  6352.                 "uid": "${DS_PROMETHEUS}"

  6353.               },

  6354.               "editorMode": "code",

  6355.               "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",

  6356.               "format": "time_series",

  6357.               "interval": "",

  6358.               "intervalFactor": 2,

  6359.               "legendFormat": "processed {{job}}-{{index}}",

  6360.               "range": true,

  6361.               "refId": "A",

  6362.               "step": 20

  6363.             },

  6364.             {

  6365.               "datasource": {

  6366.                 "uid": "${DS_PROMETHEUS}"

  6367.               },

  6368.               "editorMode": "code",

  6369.               "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",

  6370.               "format": "time_series",

  6371.               "intervalFactor": 2,

  6372.               "legendFormat": "failed {{job}}-{{index}}",

  6373.               "range": true,

  6374.               "refId": "B",

  6375.               "step": 20

  6376.             }

  6377.           ],

  6378.           "title": "HTTP Push rate",

  6379.           "type": "timeseries"

  6380.         },

  6381.         {

  6382.           "aliasColors": {},

  6383.           "bars": false,

  6384.           "dashLength": 10,

  6385.           "dashes": false,

  6386.           "datasource": {

  6387.             "uid": "${DS_PROMETHEUS}"

  6388.           },

  6389.           "description": "",

  6390.           "fieldConfig": {

  6391.             "defaults": {

  6392.               "links": []

  6393.             },

  6394.             "overrides": []

  6395.           },

  6396.           "fill": 1,

  6397.           "fillGradient": 0,

  6398.           "gridPos": {

  6399.             "h": 8,

  6400.             "w": 12,

  6401.             "x": 12,

  6402.             "y": 155

  6403.           },

  6404.           "hiddenSeries": false,

  6405.           "id": 134,

  6406.           "legend": {

  6407.             "avg": false,

  6408.             "current": false,

  6409.             "hideZero": false,

  6410.             "max": false,

  6411.             "min": false,

  6412.             "show": true,

  6413.             "total": false,

  6414.             "values": false

  6415.           },

  6416.           "lines": true,

  6417.           "linewidth": 1,

  6418.           "nullPointMode": "null",

  6419.           "options": {

  6420.             "alertThreshold": true

  6421.           },

  6422.           "percentage": false,

  6423.           "pluginVersion": "9.2.2",

  6424.           "pointradius": 2,

  6425.           "points": false,

  6426.           "renderer": "flot",

  6427.           "seriesOverrides": [],

  6428.           "spaceLength": 10,

  6429.           "stack": false,

  6430.           "steppedLine": false,

  6431.           "targets": [

  6432.             {

  6433.               "datasource": {

  6434.                 "uid": "${DS_PROMETHEUS}"

  6435.               },

  6436.               "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",

  6437.               "legendFormat": "{{kind}} {{app_id}}",

  6438.               "refId": "A"

  6439.             }

  6440.           ],

  6441.           "thresholds": [],

  6442.           "timeRegions": [],

  6443.           "title": "Active pusher instances by app",

  6444.           "tooltip": {

  6445.             "shared": false,

  6446.             "sort": 2,

  6447.             "value_type": "individual"

  6448.           },

  6449.           "type": "graph",

  6450.           "xaxis": {

  6451.             "mode": "time",

  6452.             "show": true,

  6453.             "values": []

  6454.           },

  6455.           "yaxes": [

  6456.             {

  6457.               "format": "short",

  6458.               "logBase": 1,

  6459.               "show": true

  6460.             },

  6461.             {

  6462.               "format": "short",

  6463.               "logBase": 1,

  6464.               "show": true

  6465.             }

  6466.           ],

  6467.           "yaxis": {

  6468.             "align": false

  6469.           }

  6470.         }

  6471.       ],

  6472.       "targets": [

  6473.         {

  6474.           "datasource": {

  6475.             "type": "prometheus",

  6476.             "uid": "${DS_PROMETHEUS}"

  6477.           },

  6478.           "refId": "A"

  6479.         }

  6480.       ],

  6481.       "title": "Pushes",

  6482.       "type": "row"

  6483.     },

  6484.     {

  6485.       "collapsed": true,

  6486.       "datasource": {

  6487.         "type": "prometheus",

  6488.         "uid": "${DS_PROMETHEUS}"

  6489.       },

  6490.       "gridPos": {

  6491.         "h": 1,

  6492.         "w": 24,

  6493.         "x": 0,

  6494.         "y": 33

  6495.       },

  6496.       "id": 219,

  6497.       "panels": [

  6498.         {

  6499.           "aliasColors": {},

  6500.           "bars": false,

  6501.           "dashLength": 10,

  6502.           "dashes": false,

  6503.           "datasource": {

  6504.             "type": "prometheus",

  6505.             "uid": "${DS_PROMETHEUS}"

  6506.           },

  6507.           "description": "How many entries in current state that we are iterating over while calculating push rules.",

  6508.           "fieldConfig": {

  6509.             "defaults": {

  6510.               "links": []

  6511.             },

  6512.             "overrides": []

  6513.           },

  6514.           "fill": 1,

  6515.           "fillGradient": 0,

  6516.           "gridPos": {

  6517.             "h": 7,

  6518.             "w": 12,

  6519.             "x": 0,

  6520.             "y": 33

  6521.           },

  6522.           "hiddenSeries": false,

  6523.           "id": 209,

  6524.           "legend": {

  6525.             "avg": false,

  6526.             "current": false,

  6527.             "max": false,

  6528.             "min": false,

  6529.             "show": false,

  6530.             "total": false,

  6531.             "values": false

  6532.           },

  6533.           "lines": true,

  6534.           "linewidth": 1,

  6535.           "links": [],

  6536.           "nullPointMode": "null",

  6537.           "options": {

  6538.             "alertThreshold": true

  6539.           },

  6540.           "paceLength": 10,

  6541.           "percentage": false,

  6542.           "pluginVersion": "8.4.3",

  6543.           "pointradius": 5,

  6544.           "points": false,

  6545.           "renderer": "flot",

  6546.           "seriesOverrides": [],

  6547.           "spaceLength": 10,

  6548.           "stack": false,

  6549.           "steppedLine": false,

  6550.           "targets": [

  6551.             {

  6552.               "datasource": {

  6553.                 "type": "prometheus",

  6554.                 "uid": "${DS_PROMETHEUS}"

  6555.               },

  6556.               "exemplar": true,

  6557.               "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",

  6558.               "format": "time_series",

  6559.               "interval": "",

  6560.               "intervalFactor": 2,

  6561.               "legendFormat": "{{index}}",

  6562.               "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total",

  6563.               "refId": "A",

  6564.               "step": 2

  6565.             }

  6566.           ],

  6567.           "thresholds": [],

  6568.           "timeRegions": [],

  6569.           "title": "Iterations over State",

  6570.           "tooltip": {

  6571.             "shared": true,

  6572.             "sort": 0,

  6573.             "value_type": "individual"

  6574.           },

  6575.           "type": "graph",

  6576.           "xaxis": {

  6577.             "mode": "time",

  6578.             "show": true,

  6579.             "values": []

  6580.           },

  6581.           "yaxes": [

  6582.             {

  6583.               "format": "hertz",

  6584.               "label": "",

  6585.               "logBase": 1,

  6586.               "min": "0",

  6587.               "show": true

  6588.             },

  6589.             {

  6590.               "format": "short",

  6591.               "logBase": 1,

  6592.               "show": true

  6593.             }

  6594.           ],

  6595.           "yaxis": {

  6596.             "align": false

  6597.           }

  6598.         },

  6599.         {

  6600.           "aliasColors": {},

  6601.           "bars": false,

  6602.           "dashLength": 10,

  6603.           "dashes": false,

  6604.           "datasource": {

  6605.             "type": "prometheus",

  6606.             "uid": "${DS_PROMETHEUS}"

  6607.           },

  6608.           "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ",

  6609.           "fieldConfig": {

  6610.             "defaults": {

  6611.               "links": []

  6612.             },

  6613.             "overrides": []

  6614.           },

  6615.           "fill": 1,

  6616.           "fillGradient": 0,

  6617.           "gridPos": {

  6618.             "h": 7,

  6619.             "w": 12,

  6620.             "x": 12,

  6621.             "y": 33

  6622.           },

  6623.           "hiddenSeries": false,

  6624.           "id": 211,

  6625.           "legend": {

  6626.             "avg": false,

  6627.             "current": false,

  6628.             "max": false,

  6629.             "min": false,

  6630.             "show": false,

  6631.             "total": false,

  6632.             "values": false

  6633.           },

  6634.           "lines": true,

  6635.           "linewidth": 1,

  6636.           "links": [],

  6637.           "nullPointMode": "null",

  6638.           "options": {

  6639.             "alertThreshold": true

  6640.           },

  6641.           "paceLength": 10,

  6642.           "percentage": false,

  6643.           "pluginVersion": "8.4.3",

  6644.           "pointradius": 5,

  6645.           "points": false,

  6646.           "renderer": "flot",

  6647.           "seriesOverrides": [],

  6648.           "spaceLength": 10,

  6649.           "stack": false,

  6650.           "steppedLine": false,

  6651.           "targets": [

  6652.             {

  6653.               "datasource": {

  6654.                 "type": "prometheus",

  6655.                 "uid": "${DS_PROMETHEUS}"

  6656.               },

  6657.               "exemplar": true,

  6658.               "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",

  6659.               "format": "time_series",

  6660.               "interval": "",

  6661.               "intervalFactor": 2,

  6662.               "legendFormat": "{{index}}",

  6663.               "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",

  6664.               "refId": "A",

  6665.               "step": 2

  6666.             }

  6667.           ],

  6668.           "thresholds": [],

  6669.           "timeRegions": [],

  6670.           "title": "Push Rule Invalidations",

  6671.           "tooltip": {

  6672.             "shared": true,

  6673.             "sort": 0,

  6674.             "value_type": "individual"

  6675.           },

  6676.           "type": "graph",

  6677.           "xaxis": {

  6678.             "mode": "time",

  6679.             "show": true,

  6680.             "values": []

  6681.           },

  6682.           "yaxes": [

  6683.             {

  6684.               "format": "hertz",

  6685.               "label": "",

  6686.               "logBase": 1,

  6687.               "min": "0",

  6688.               "show": true

  6689.             },

  6690.             {

  6691.               "format": "short",

  6692.               "logBase": 1,

  6693.               "show": true

  6694.             }

  6695.           ],

  6696.           "yaxis": {

  6697.             "align": false

  6698.           }

  6699.         },

  6700.         {

  6701.           "aliasColors": {},

  6702.           "bars": false,

  6703.           "dashLength": 10,

  6704.           "dashes": false,

  6705.           "datasource": {

  6706.             "type": "prometheus",

  6707.             "uid": "${DS_PROMETHEUS}"

  6708.           },

  6709.           "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.",

  6710.           "fieldConfig": {

  6711.             "defaults": {

  6712.               "links": []

  6713.             },

  6714.             "overrides": []

  6715.           },

  6716.           "fill": 1,

  6717.           "fillGradient": 0,

  6718.           "gridPos": {

  6719.             "h": 7,

  6720.             "w": 12,

  6721.             "x": 0,

  6722.             "y": 40

  6723.           },

  6724.           "hiddenSeries": false,

  6725.           "id": 213,

  6726.           "legend": {

  6727.             "avg": false,

  6728.             "current": false,

  6729.             "max": false,

  6730.             "min": false,

  6731.             "show": true,

  6732.             "total": false,

  6733.             "values": false

  6734.           },

  6735.           "lines": true,

  6736.           "linewidth": 1,

  6737.           "links": [],

  6738.           "nullPointMode": "null",

  6739.           "options": {

  6740.             "alertThreshold": true

  6741.           },

  6742.           "paceLength": 10,

  6743.           "percentage": false,

  6744.           "pluginVersion": "8.4.3",

  6745.           "pointradius": 5,

  6746.           "points": false,

  6747.           "renderer": "flot",

  6748.           "seriesOverrides": [

  6749.             {

  6750.               "alias": "Number of calls",

  6751.               "yaxis": 2

  6752.             }

  6753.           ],

  6754.           "spaceLength": 10,

  6755.           "stack": false,

  6756.           "steppedLine": false,

  6757.           "targets": [

  6758.             {

  6759.               "datasource": {

  6760.                 "type": "prometheus",

  6761.                 "uid": "${DS_PROMETHEUS}"

  6762.               },

  6763.               "exemplar": true,

  6764.               "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",

  6765.               "format": "time_series",

  6766.               "interval": "",

  6767.               "intervalFactor": 2,

  6768.               "legendFormat": "Hit Rate",

  6769.               "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",

  6770.               "refId": "A",

  6771.               "step": 2

  6772.             },

  6773.             {

  6774.               "datasource": {

  6775.                 "type": "prometheus",

  6776.                 "uid": "${DS_PROMETHEUS}"

  6777.               },

  6778.               "exemplar": true,

  6779.               "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",

  6780.               "format": "time_series",

  6781.               "interval": "",

  6782.               "intervalFactor": 2,

  6783.               "legendFormat": "Number of calls",

  6784.               "refId": "B",

  6785.               "step": 2

  6786.             }

  6787.           ],

  6788.           "thresholds": [],

  6789.           "timeRegions": [],

  6790.           "title": "Delta Optimisation",

  6791.           "tooltip": {

  6792.             "shared": true,

  6793.             "sort": 0,

  6794.             "value_type": "individual"

  6795.           },

  6796.           "type": "graph",

  6797.           "xaxis": {

  6798.             "mode": "time",

  6799.             "show": true,

  6800.             "values": []

  6801.           },

  6802.           "yaxes": [

  6803.             {

  6804.               "format": "percentunit",

  6805.               "label": "",

  6806.               "logBase": 1,

  6807.               "max": "1",

  6808.               "min": "0",

  6809.               "show": true

  6810.             },

  6811.             {

  6812.               "format": "hertz",

  6813.               "label": "",

  6814.               "logBase": 1,

  6815.               "min": "0",

  6816.               "show": true

  6817.             }

  6818.           ],

  6819.           "yaxis": {

  6820.             "align": false

  6821.           }

  6822.         },

  6823.         {

  6824.           "aliasColors": {},

  6825.           "bars": false,

  6826.           "dashLength": 10,

  6827.           "dashes": false,

  6828.           "datasource": {

  6829.             "type": "prometheus",

  6830.             "uid": "${DS_PROMETHEUS}"

  6831.           },

  6832.           "description": "How often we have the correct cached push rules for a room.",

  6833.           "fieldConfig": {

  6834.             "defaults": {

  6835.               "links": []

  6836.             },

  6837.             "overrides": []

  6838.           },

  6839.           "fill": 1,

  6840.           "fillGradient": 0,

  6841.           "gridPos": {

  6842.             "h": 7,

  6843.             "w": 12,

  6844.             "x": 12,

  6845.             "y": 40

  6846.           },

  6847.           "hiddenSeries": false,

  6848.           "id": 215,

  6849.           "legend": {

  6850.             "avg": false,

  6851.             "current": false,

  6852.             "max": false,

  6853.             "min": false,

  6854.             "show": true,

  6855.             "total": false,

  6856.             "values": false

  6857.           },

  6858.           "lines": true,

  6859.           "linewidth": 1,

  6860.           "links": [],

  6861.           "nullPointMode": "null",

  6862.           "options": {

  6863.             "alertThreshold": true

  6864.           },

  6865.           "paceLength": 10,

  6866.           "percentage": false,

  6867.           "pluginVersion": "8.4.3",

  6868.           "pointradius": 5,

  6869.           "points": false,

  6870.           "renderer": "flot",

  6871.           "seriesOverrides": [

  6872.             {

  6873.               "alias": "Number of calls",

  6874.               "yaxis": 2

  6875.             }

  6876.           ],

  6877.           "spaceLength": 10,

  6878.           "stack": false,

  6879.           "steppedLine": false,

  6880.           "targets": [

  6881.             {

  6882.               "datasource": {

  6883.                 "type": "prometheus",

  6884.                 "uid": "${DS_PROMETHEUS}"

  6885.               },

  6886.               "exemplar": true,

  6887.               "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",

  6888.               "format": "time_series",

  6889.               "interval": "",

  6890.               "intervalFactor": 2,

  6891.               "legendFormat": "Hit Rate",

  6892.               "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",

  6893.               "refId": "A",

  6894.               "step": 2

  6895.             },

  6896.             {

  6897.               "datasource": {

  6898.                 "type": "prometheus",

  6899.                 "uid": "${DS_PROMETHEUS}"

  6900.               },

  6901.               "exemplar": true,

  6902.               "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",

  6903.               "format": "time_series",

  6904.               "interval": "",

  6905.               "intervalFactor": 2,

  6906.               "legendFormat": "Number of calls",

  6907.               "refId": "B",

  6908.               "step": 2

  6909.             }

  6910.           ],

  6911.           "thresholds": [],

  6912.           "timeRegions": [],

  6913.           "title": "How often we reuse existing calculated push rules",

  6914.           "tooltip": {

  6915.             "shared": true,

  6916.             "sort": 0,

  6917.             "value_type": "individual"

  6918.           },

  6919.           "type": "graph",

  6920.           "xaxis": {

  6921.             "mode": "time",

  6922.             "show": true,

  6923.             "values": []

  6924.           },

  6925.           "yaxes": [

  6926.             {

  6927.               "format": "percentunit",

  6928.               "label": "",

  6929.               "logBase": 1,

  6930.               "max": "1",

  6931.               "min": "0",

  6932.               "show": true

  6933.             },

  6934.             {

  6935.               "format": "hertz",

  6936.               "logBase": 1,

  6937.               "show": true

  6938.             }

  6939.           ],

  6940.           "yaxis": {

  6941.             "align": false

  6942.           }

  6943.         },

  6944.         {

  6945.           "aliasColors": {},

  6946.           "bars": false,

  6947.           "dashLength": 10,

  6948.           "dashes": false,

  6949.           "datasource": {

  6950.             "type": "prometheus",

  6951.             "uid": "${DS_PROMETHEUS}"

  6952.           },

  6953.           "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.",

  6954.           "fieldConfig": {

  6955.             "defaults": {

  6956.               "links": []

  6957.             },

  6958.             "overrides": []

  6959.           },

  6960.           "fill": 1,

  6961.           "fillGradient": 0,

  6962.           "gridPos": {

  6963.             "h": 7,

  6964.             "w": 12,

  6965.             "x": 0,

  6966.             "y": 47

  6967.           },

  6968.           "hiddenSeries": false,

  6969.           "id": 217,

  6970.           "legend": {

  6971.             "avg": false,

  6972.             "current": false,

  6973.             "max": false,

  6974.             "min": false,

  6975.             "show": true,

  6976.             "total": false,

  6977.             "values": false

  6978.           },

  6979.           "lines": true,

  6980.           "linewidth": 1,

  6981.           "links": [],

  6982.           "nullPointMode": "null",

  6983.           "options": {

  6984.             "alertThreshold": true

  6985.           },

  6986.           "paceLength": 10,

  6987.           "percentage": false,

  6988.           "pluginVersion": "8.4.3",

  6989.           "pointradius": 5,

  6990.           "points": false,

  6991.           "renderer": "flot",

  6992.           "seriesOverrides": [

  6993.             {

  6994.               "alias": "Number of calls",

  6995.               "yaxis": 2

  6996.             }

  6997.           ],

  6998.           "spaceLength": 10,

  6999.           "stack": false,

  7000.           "steppedLine": false,

  7001.           "targets": [

  7002.             {

  7003.               "datasource": {

  7004.                 "type": "prometheus",

  7005.                 "uid": "${DS_PROMETHEUS}"

  7006.               },

  7007.               "exemplar": true,

  7008.               "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",

  7009.               "format": "time_series",

  7010.               "interval": "",

  7011.               "intervalFactor": 2,

  7012.               "legendFormat": "Hit Rate",

  7013.               "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",

  7014.               "refId": "A",

  7015.               "step": 2

  7016.             },

  7017.             {

  7018.               "datasource": {

  7019.                 "type": "prometheus",

  7020.                 "uid": "${DS_PROMETHEUS}"

  7021.               },

  7022.               "exemplar": true,

  7023.               "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",

  7024.               "format": "time_series",

  7025.               "interval": "",

  7026.               "intervalFactor": 2,

  7027.               "legendFormat": "Number of calls",

  7028.               "refId": "B",

  7029.               "step": 2

  7030.             }

  7031.           ],

  7032.           "thresholds": [],

  7033.           "timeRegions": [],

  7034.           "title": "How often we have the RulesForRoom cached",

  7035.           "tooltip": {

  7036.             "shared": true,

  7037.             "sort": 0,

  7038.             "value_type": "individual"

  7039.           },

  7040.           "type": "graph",

  7041.           "xaxis": {

  7042.             "mode": "time",

  7043.             "show": true,

  7044.             "values": []

  7045.           },

  7046.           "yaxes": [

  7047.             {

  7048.               "format": "percentunit",

  7049.               "label": "",

  7050.               "logBase": 1,

  7051.               "max": "1",

  7052.               "min": "0",

  7053.               "show": true

  7054.             },

  7055.             {

  7056.               "format": "hertz",

  7057.               "logBase": 1,

  7058.               "show": true

  7059.             }

  7060.           ],

  7061.           "yaxis": {

  7062.             "align": false

  7063.           }

  7064.         }

  7065.       ],

  7066.       "targets": [

  7067.         {

  7068.           "datasource": {

  7069.             "type": "prometheus",

  7070.             "uid": "${DS_PROMETHEUS}"

  7071.           },

  7072.           "refId": "A"

  7073.         }

  7074.       ],

  7075.       "title": "Push Rule Cache",

  7076.       "type": "row"

  7077.     },

  7078.     {

  7079.       "collapsed": true,

  7080.       "datasource": {

  7081.         "type": "prometheus",

  7082.         "uid": "${DS_PROMETHEUS}"

  7083.       },

  7084.       "gridPos": {

  7085.         "h": 1,

  7086.         "w": 24,

  7087.         "x": 0,

  7088.         "y": 34

  7089.       },

  7090.       "id": 58,

  7091.       "panels": [

  7092.         {

  7093.           "datasource": {

  7094.             "uid": "${DS_PROMETHEUS}"

  7095.           },

  7096.           "fieldConfig": {

  7097.             "defaults": {

  7098.               "color": {

  7099.                 "mode": "palette-classic"

  7100.               },

  7101.               "custom": {

  7102.                 "axisLabel": "",

  7103.                 "axisPlacement": "auto",

  7104.                 "barAlignment": 0,

  7105.                 "drawStyle": "line",

  7106.                 "fillOpacity": 10,

  7107.                 "gradientMode": "none",

  7108.                 "hideFrom": {

  7109.                   "legend": false,

  7110.                   "tooltip": false,

  7111.                   "viz": false

  7112.                 },

  7113.                 "lineInterpolation": "linear",

  7114.                 "lineWidth": 1,

  7115.                 "pointSize": 5,

  7116.                 "scaleDistribution": {

  7117.                   "type": "linear"

  7118.                 },

  7119.                 "showPoints": "never",

  7120.                 "spanNulls": false,

  7121.                 "stacking": {

  7122.                   "group": "A",

  7123.                   "mode": "none"

  7124.                 },

  7125.                 "thresholdsStyle": {

  7126.                   "mode": "off"

  7127.                 }

  7128.               },

  7129.               "links": [],

  7130.               "mappings": [],

  7131.               "min": 0,

  7132.               "thresholds": {

  7133.                 "mode": "absolute",

  7134.                 "steps": [

  7135.                   {

  7136.                     "color": "green"

  7137.                   },

  7138.                   {

  7139.                     "color": "red",

  7140.                     "value": 80

  7141.                   }

  7142.                 ]

  7143.               },

  7144.               "unit": "s"

  7145.             },

  7146.             "overrides": []

  7147.           },

  7148.           "gridPos": {

  7149.             "h": 7,

  7150.             "w": 12,

  7151.             "x": 0,

  7152.             "y": 35

  7153.           },

  7154.           "id": 48,

  7155.           "links": [],

  7156.           "options": {

  7157.             "legend": {

  7158.               "calcs": [],

  7159.               "displayMode": "list",

  7160.               "placement": "bottom",

  7161.               "showLegend": true

  7162.             },

  7163.             "tooltip": {

  7164.               "mode": "single",

  7165.               "sort": "none"

  7166.             }

  7167.           },

  7168.           "pluginVersion": "9.0.4",

  7169.           "targets": [

  7170.             {

  7171.               "datasource": {

  7172.                 "uid": "${DS_PROMETHEUS}"

  7173.               },

  7174.               "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",

  7175.               "format": "time_series",

  7176.               "intervalFactor": 2,

  7177.               "legendFormat": "{{job}}-{{index}}",

  7178.               "refId": "A",

  7179.               "step": 20

  7180.             }

  7181.           ],

  7182.           "title": "Avg time waiting for db conn",

  7183.           "type": "timeseries"

  7184.         },

  7185.         {

  7186.           "aliasColors": {},

  7187.           "bars": false,

  7188.           "dashLength": 10,

  7189.           "dashes": false,

  7190.           "datasource": {

  7191.             "uid": "${DS_PROMETHEUS}"

  7192.           },

  7193.           "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",

  7194.           "fieldConfig": {

  7195.             "defaults": {

  7196.               "links": []

  7197.             },

  7198.             "overrides": []

  7199.           },

  7200.           "fill": 1,

  7201.           "fillGradient": 0,

  7202.           "gridPos": {

  7203.             "h": 7,

  7204.             "w": 12,

  7205.             "x": 12,

  7206.             "y": 35

  7207.           },

  7208.           "hiddenSeries": false,

  7209.           "id": 104,

  7210.           "legend": {

  7211.             "alignAsTable": true,

  7212.             "avg": false,

  7213.             "current": false,

  7214.             "max": false,

  7215.             "min": false,

  7216.             "show": true,

  7217.             "total": false,

  7218.             "values": false

  7219.           },

  7220.           "lines": true,

  7221.           "linewidth": 1,

  7222.           "links": [],

  7223.           "nullPointMode": "null",

  7224.           "options": {

  7225.             "alertThreshold": true

  7226.           },

  7227.           "paceLength": 10,

  7228.           "percentage": false,

  7229.           "pluginVersion": "9.0.4",

  7230.           "pointradius": 5,

  7231.           "points": false,

  7232.           "renderer": "flot",

  7233.           "seriesOverrides": [],

  7234.           "spaceLength": 10,

  7235.           "stack": false,

  7236.           "steppedLine": false,

  7237.           "targets": [

  7238.             {

  7239.               "datasource": {

  7240.                 "uid": "${DS_PROMETHEUS}"

  7241.               },

  7242.               "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  7243.               "format": "time_series",

  7244.               "hide": false,

  7245.               "intervalFactor": 1,

  7246.               "legendFormat": "{{job}} {{index}} 99%",

  7247.               "refId": "A",

  7248.               "step": 20

  7249.             },

  7250.             {

  7251.               "datasource": {

  7252.                 "uid": "${DS_PROMETHEUS}"

  7253.               },

  7254.               "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  7255.               "format": "time_series",

  7256.               "intervalFactor": 1,

  7257.               "legendFormat": "{{job}} {{index}} 95%",

  7258.               "refId": "B"

  7259.             },

  7260.             {

  7261.               "datasource": {

  7262.                 "uid": "${DS_PROMETHEUS}"

  7263.               },

  7264.               "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  7265.               "format": "time_series",

  7266.               "intervalFactor": 1,

  7267.               "legendFormat": "{{job}} {{index}} 90%",

  7268.               "refId": "C"

  7269.             },

  7270.             {

  7271.               "datasource": {

  7272.                 "uid": "${DS_PROMETHEUS}"

  7273.               },

  7274.               "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  7275.               "format": "time_series",

  7276.               "interval": "",

  7277.               "intervalFactor": 1,

  7278.               "legendFormat": "{{job}} {{index}} mean",

  7279.               "refId": "D"

  7280.             }

  7281.           ],

  7282.           "thresholds": [],

  7283.           "timeRegions": [],

  7284.           "title": "Db scheduling time quantiles",

  7285.           "tooltip": {

  7286.             "shared": false,

  7287.             "sort": 0,

  7288.             "value_type": "individual"

  7289.           },

  7290.           "type": "graph",

  7291.           "xaxis": {

  7292.             "mode": "time",

  7293.             "show": true,

  7294.             "values": []

  7295.           },

  7296.           "yaxes": [

  7297.             {

  7298.               "format": "s",

  7299.               "label": "",

  7300.               "logBase": 1,

  7301.               "min": "0",

  7302.               "show": true

  7303.             },

  7304.             {

  7305.               "format": "short",

  7306.               "logBase": 1,

  7307.               "show": false

  7308.             }

  7309.           ],

  7310.           "yaxis": {

  7311.             "align": false

  7312.           }

  7313.         },

  7314.         {

  7315.           "aliasColors": {},

  7316.           "bars": false,

  7317.           "dashLength": 10,

  7318.           "dashes": false,

  7319.           "datasource": {

  7320.             "uid": "${DS_PROMETHEUS}"

  7321.           },

  7322.           "editable": true,

  7323.           "error": false,

  7324.           "fieldConfig": {

  7325.             "defaults": {

  7326.               "links": []

  7327.             },

  7328.             "overrides": []

  7329.           },

  7330.           "fill": 0,

  7331.           "fillGradient": 0,

  7332.           "grid": {},

  7333.           "gridPos": {

  7334.             "h": 7,

  7335.             "w": 12,

  7336.             "x": 0,

  7337.             "y": 42

  7338.           },

  7339.           "hiddenSeries": false,

  7340.           "id": 10,

  7341.           "legend": {

  7342.             "avg": false,

  7343.             "current": false,

  7344.             "hideEmpty": true,

  7345.             "hideZero": true,

  7346.             "max": false,

  7347.             "min": false,

  7348.             "show": true,

  7349.             "total": false,

  7350.             "values": false

  7351.           },

  7352.           "lines": true,

  7353.           "linewidth": 2,

  7354.           "links": [],

  7355.           "nullPointMode": "null",

  7356.           "options": {

  7357.             "alertThreshold": true

  7358.           },

  7359.           "paceLength": 10,

  7360.           "percentage": false,

  7361.           "pluginVersion": "9.0.4",

  7362.           "pointradius": 5,

  7363.           "points": false,

  7364.           "renderer": "flot",

  7365.           "seriesOverrides": [],

  7366.           "spaceLength": 10,

  7367.           "stack": false,

  7368.           "steppedLine": false,

  7369.           "targets": [

  7370.             {

  7371.               "datasource": {

  7372.                 "uid": "${DS_PROMETHEUS}"

  7373.               },

  7374.               "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  7375.               "format": "time_series",

  7376.               "interval": "",

  7377.               "intervalFactor": 2,

  7378.               "legendFormat": "{{job}}-{{index}} {{desc}}",

  7379.               "refId": "A",

  7380.               "step": 20

  7381.             }

  7382.           ],

  7383.           "thresholds": [],

  7384.           "timeRegions": [],

  7385.           "title": "Top DB transactions by txn rate",

  7386.           "tooltip": {

  7387.             "shared": false,

  7388.             "sort": 0,

  7389.             "value_type": "cumulative"

  7390.           },

  7391.           "type": "graph",

  7392.           "xaxis": {

  7393.             "mode": "time",

  7394.             "show": true,

  7395.             "values": []

  7396.           },

  7397.           "yaxes": [

  7398.             {

  7399.               "format": "hertz",

  7400.               "logBase": 1,

  7401.               "min": 0,

  7402.               "show": true

  7403.             },

  7404.             {

  7405.               "format": "short",

  7406.               "logBase": 1,

  7407.               "show": true

  7408.             }

  7409.           ],

  7410.           "yaxis": {

  7411.             "align": false

  7412.           }

  7413.         },

  7414.         {

  7415.           "aliasColors": {},

  7416.           "bars": false,

  7417.           "dashLength": 10,

  7418.           "dashes": false,

  7419.           "datasource": {

  7420.             "uid": "${DS_PROMETHEUS}"

  7421.           },

  7422.           "editable": true,

  7423.           "error": false,

  7424.           "fieldConfig": {

  7425.             "defaults": {

  7426.               "links": []

  7427.             },

  7428.             "overrides": []

  7429.           },

  7430.           "fill": 1,

  7431.           "fillGradient": 0,

  7432.           "grid": {},

  7433.           "gridPos": {

  7434.             "h": 7,

  7435.             "w": 12,

  7436.             "x": 12,

  7437.             "y": 42

  7438.           },

  7439.           "hiddenSeries": false,

  7440.           "id": 11,

  7441.           "legend": {

  7442.             "avg": false,

  7443.             "current": false,

  7444.             "hideEmpty": true,

  7445.             "hideZero": true,

  7446.             "max": false,

  7447.             "min": false,

  7448.             "show": true,

  7449.             "total": false,

  7450.             "values": false

  7451.           },

  7452.           "lines": true,

  7453.           "linewidth": 1,

  7454.           "links": [],

  7455.           "nullPointMode": "null",

  7456.           "options": {

  7457.             "alertThreshold": true

  7458.           },

  7459.           "paceLength": 10,

  7460.           "percentage": false,

  7461.           "pluginVersion": "9.0.4",

  7462.           "pointradius": 5,

  7463.           "points": false,

  7464.           "renderer": "flot",

  7465.           "seriesOverrides": [],

  7466.           "spaceLength": 10,

  7467.           "stack": false,

  7468.           "steppedLine": false,

  7469.           "targets": [

  7470.             {

  7471.               "datasource": {

  7472.                 "uid": "${DS_PROMETHEUS}"

  7473.               },

  7474.               "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  7475.               "format": "time_series",

  7476.               "instant": false,

  7477.               "interval": "",

  7478.               "intervalFactor": 1,

  7479.               "legendFormat": "{{job}}-{{index}} {{desc}}",

  7480.               "refId": "A",

  7481.               "step": 20

  7482.             }

  7483.           ],

  7484.           "thresholds": [],

  7485.           "timeRegions": [],

  7486.           "title": "DB transactions by total txn time",

  7487.           "tooltip": {

  7488.             "shared": false,

  7489.             "sort": 0,

  7490.             "value_type": "cumulative"

  7491.           },

  7492.           "type": "graph",

  7493.           "xaxis": {

  7494.             "mode": "time",

  7495.             "show": true,

  7496.             "values": []

  7497.           },

  7498.           "yaxes": [

  7499.             {

  7500.               "format": "percentunit",

  7501.               "logBase": 1,

  7502.               "show": true

  7503.             },

  7504.             {

  7505.               "format": "short",

  7506.               "logBase": 1,

  7507.               "show": true

  7508.             }

  7509.           ],

  7510.           "yaxis": {

  7511.             "align": false

  7512.           }

  7513.         },

  7514.         {

  7515.           "aliasColors": {},

  7516.           "bars": false,

  7517.           "dashLength": 10,

  7518.           "dashes": false,

  7519.           "datasource": {

  7520.             "uid": "${DS_PROMETHEUS}"

  7521.           },

  7522.           "editable": true,

  7523.           "error": false,

  7524.           "fieldConfig": {

  7525.             "defaults": {

  7526.               "links": []

  7527.             },

  7528.             "overrides": []

  7529.           },

  7530.           "fill": 0,

  7531.           "fillGradient": 0,

  7532.           "grid": {},

  7533.           "gridPos": {

  7534.             "h": 7,

  7535.             "w": 12,

  7536.             "x": 0,

  7537.             "y": 49

  7538.           },

  7539.           "hiddenSeries": false,

  7540.           "id": 180,

  7541.           "legend": {

  7542.             "avg": false,

  7543.             "current": false,

  7544.             "hideEmpty": true,

  7545.             "hideZero": true,

  7546.             "max": false,

  7547.             "min": false,

  7548.             "show": true,

  7549.             "total": false,

  7550.             "values": false

  7551.           },

  7552.           "lines": true,

  7553.           "linewidth": 1,

  7554.           "links": [],

  7555.           "nullPointMode": "null",

  7556.           "options": {

  7557.             "alertThreshold": false

  7558.           },

  7559.           "paceLength": 10,

  7560.           "percentage": false,

  7561.           "pluginVersion": "9.0.4",

  7562.           "pointradius": 5,

  7563.           "points": false,

  7564.           "renderer": "flot",

  7565.           "seriesOverrides": [],

  7566.           "spaceLength": 10,

  7567.           "stack": false,

  7568.           "steppedLine": false,

  7569.           "targets": [

  7570.             {

  7571.               "datasource": {

  7572.                 "uid": "${DS_PROMETHEUS}"

  7573.               },

  7574.               "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  7575.               "format": "time_series",

  7576.               "instant": false,

  7577.               "interval": "",

  7578.               "intervalFactor": 1,

  7579.               "legendFormat": "{{job}}-{{index}} {{desc}}",

  7580.               "refId": "A",

  7581.               "step": 20

  7582.             }

  7583.           ],

  7584.           "thresholds": [],

  7585.           "timeRegions": [],

  7586.           "title": "Average DB txn time",

  7587.           "tooltip": {

  7588.             "shared": false,

  7589.             "sort": 0,

  7590.             "value_type": "cumulative"

  7591.           },

  7592.           "type": "graph",

  7593.           "xaxis": {

  7594.             "mode": "time",

  7595.             "show": true,

  7596.             "values": []

  7597.           },

  7598.           "yaxes": [

  7599.             {

  7600.               "format": "s",

  7601.               "logBase": 1,

  7602.               "show": true

  7603.             },

  7604.             {

  7605.               "format": "short",

  7606.               "logBase": 1,

  7607.               "show": true

  7608.             }

  7609.           ],

  7610.           "yaxis": {

  7611.             "align": false

  7612.           }

  7613.         },

  7614.         {

  7615.           "aliasColors": {},

  7616.           "bars": false,

  7617.           "dashLength": 10,

  7618.           "dashes": false,

  7619.           "datasource": {

  7620.             "uid": "${DS_PROMETHEUS}"

  7621.           },

  7622.           "fieldConfig": {

  7623.             "defaults": {

  7624.               "links": []

  7625.             },

  7626.             "overrides": []

  7627.           },

  7628.           "fill": 6,

  7629.           "fillGradient": 0,

  7630.           "gridPos": {

  7631.             "h": 9,

  7632.             "w": 12,

  7633.             "x": 12,

  7634.             "y": 49

  7635.           },

  7636.           "hiddenSeries": false,

  7637.           "id": 200,

  7638.           "legend": {

  7639.             "avg": false,

  7640.             "current": false,

  7641.             "max": false,

  7642.             "min": false,

  7643.             "show": true,

  7644.             "total": false,

  7645.             "values": false

  7646.           },

  7647.           "lines": true,

  7648.           "linewidth": 1,

  7649.           "links": [],

  7650.           "nullPointMode": "null",

  7651.           "options": {

  7652.             "alertThreshold": true

  7653.           },

  7654.           "percentage": false,

  7655.           "pluginVersion": "9.0.4",

  7656.           "pointradius": 5,

  7657.           "points": false,

  7658.           "renderer": "flot",

  7659.           "seriesOverrides": [],

  7660.           "spaceLength": 10,

  7661.           "stack": false,

  7662.           "steppedLine": false,

  7663.           "targets": [

  7664.             {

  7665.               "datasource": {

  7666.                 "uid": "${DS_PROMETHEUS}"

  7667.               },

  7668.               "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",

  7669.               "format": "time_series",

  7670.               "intervalFactor": 1,

  7671.               "legendFormat": "99%",

  7672.               "refId": "D"

  7673.             },

  7674.             {

  7675.               "datasource": {

  7676.                 "uid": "${DS_PROMETHEUS}"

  7677.               },

  7678.               "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",

  7679.               "format": "time_series",

  7680.               "intervalFactor": 1,

  7681.               "legendFormat": "90%",

  7682.               "refId": "A"

  7683.             },

  7684.             {

  7685.               "datasource": {

  7686.                 "uid": "${DS_PROMETHEUS}"

  7687.               },

  7688.               "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",

  7689.               "format": "time_series",

  7690.               "intervalFactor": 1,

  7691.               "legendFormat": "75%",

  7692.               "refId": "C"

  7693.             },

  7694.             {

  7695.               "datasource": {

  7696.                 "uid": "${DS_PROMETHEUS}"

  7697.               },

  7698.               "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",

  7699.               "format": "time_series",

  7700.               "intervalFactor": 1,

  7701.               "legendFormat": "50%",

  7702.               "refId": "B"

  7703.             }

  7704.           ],

  7705.           "thresholds": [],

  7706.           "timeRegions": [],

  7707.           "title": "Time waiting for DB connection quantiles",

  7708.           "tooltip": {

  7709.             "shared": true,

  7710.             "sort": 2,

  7711.             "value_type": "individual"

  7712.           },

  7713.           "type": "graph",

  7714.           "xaxis": {

  7715.             "mode": "time",

  7716.             "show": true,

  7717.             "values": []

  7718.           },

  7719.           "yaxes": [

  7720.             {

  7721.               "$$hashKey": "object:203",

  7722.               "format": "s",

  7723.               "label": "",

  7724.               "logBase": 1,

  7725.               "min": "0",

  7726.               "show": true

  7727.             },

  7728.             {

  7729.               "$$hashKey": "object:204",

  7730.               "format": "short",

  7731.               "logBase": 1,

  7732.               "show": false

  7733.             }

  7734.           ],

  7735.           "yaxis": {

  7736.             "align": false

  7737.           }

  7738.         }

  7739.       ],

  7740.       "targets": [

  7741.         {

  7742.           "datasource": {

  7743.             "type": "prometheus",

  7744.             "uid": "${DS_PROMETHEUS}"

  7745.           },

  7746.           "refId": "A"

  7747.         }

  7748.       ],

  7749.       "title": "Database",

  7750.       "type": "row"

  7751.     },

  7752.     {

  7753.       "collapsed": true,

  7754.       "datasource": {

  7755.         "type": "prometheus",

  7756.         "uid": "${DS_PROMETHEUS}"

  7757.       },

  7758.       "gridPos": {

  7759.         "h": 1,

  7760.         "w": 24,

  7761.         "x": 0,

  7762.         "y": 35

  7763.       },

  7764.       "id": 59,

  7765.       "panels": [

  7766.         {

  7767.           "aliasColors": {},

  7768.           "bars": false,

  7769.           "dashLength": 10,

  7770.           "dashes": false,

  7771.           "datasource": {

  7772.             "uid": "${DS_PROMETHEUS}"

  7773.           },

  7774.           "editable": true,

  7775.           "error": false,

  7776.           "fieldConfig": {

  7777.             "defaults": {

  7778.               "links": []

  7779.             },

  7780.             "overrides": []

  7781.           },

  7782.           "fill": 1,

  7783.           "fillGradient": 0,

  7784.           "grid": {},

  7785.           "gridPos": {

  7786.             "h": 13,

  7787.             "w": 12,

  7788.             "x": 0,

  7789.             "y": 158

  7790.           },

  7791.           "hiddenSeries": false,

  7792.           "id": 12,

  7793.           "legend": {

  7794.             "alignAsTable": true,

  7795.             "avg": false,

  7796.             "current": false,

  7797.             "max": false,

  7798.             "min": false,

  7799.             "show": true,

  7800.             "total": false,

  7801.             "values": false

  7802.           },

  7803.           "lines": true,

  7804.           "linewidth": 2,

  7805.           "links": [],

  7806.           "nullPointMode": "null",

  7807.           "options": {

  7808.             "alertThreshold": true

  7809.           },

  7810.           "paceLength": 10,

  7811.           "percentage": false,

  7812.           "pluginVersion": "9.2.2",

  7813.           "pointradius": 5,

  7814.           "points": false,

  7815.           "renderer": "flot",

  7816.           "seriesOverrides": [],

  7817.           "spaceLength": 10,

  7818.           "stack": false,

  7819.           "steppedLine": false,

  7820.           "targets": [

  7821.             {

  7822.               "datasource": {

  7823.                 "uid": "${DS_PROMETHEUS}"

  7824.               },

  7825.               "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])",

  7826.               "format": "time_series",

  7827.               "interval": "",

  7828.               "intervalFactor": 2,

  7829.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  7830.               "refId": "A",

  7831.               "step": 20

  7832.             }

  7833.           ],

  7834.           "thresholds": [],

  7835.           "timeRegions": [],

  7836.           "title": "Total CPU Usage by Block",

  7837.           "tooltip": {

  7838.             "shared": true,

  7839.             "sort": 2,

  7840.             "value_type": "cumulative"

  7841.           },

  7842.           "type": "graph",

  7843.           "xaxis": {

  7844.             "mode": "time",

  7845.             "show": true,

  7846.             "values": []

  7847.           },

  7848.           "yaxes": [

  7849.             {

  7850.               "format": "percentunit",

  7851.               "logBase": 1,

  7852.               "show": true

  7853.             },

  7854.             {

  7855.               "format": "short",

  7856.               "logBase": 1,

  7857.               "show": true

  7858.             }

  7859.           ],

  7860.           "yaxis": {

  7861.             "align": false

  7862.           }

  7863.         },

  7864.         {

  7865.           "aliasColors": {},

  7866.           "bars": false,

  7867.           "dashLength": 10,

  7868.           "dashes": false,

  7869.           "datasource": {

  7870.             "uid": "${DS_PROMETHEUS}"

  7871.           },

  7872.           "editable": true,

  7873.           "error": false,

  7874.           "fieldConfig": {

  7875.             "defaults": {

  7876.               "links": []

  7877.             },

  7878.             "overrides": []

  7879.           },

  7880.           "fill": 1,

  7881.           "fillGradient": 0,

  7882.           "grid": {},

  7883.           "gridPos": {

  7884.             "h": 13,

  7885.             "w": 12,

  7886.             "x": 12,

  7887.             "y": 158

  7888.           },

  7889.           "hiddenSeries": false,

  7890.           "id": 26,

  7891.           "legend": {

  7892.             "alignAsTable": true,

  7893.             "avg": false,

  7894.             "current": false,

  7895.             "max": false,

  7896.             "min": false,

  7897.             "show": true,

  7898.             "total": false,

  7899.             "values": false

  7900.           },

  7901.           "lines": true,

  7902.           "linewidth": 2,

  7903.           "links": [],

  7904.           "nullPointMode": "null",

  7905.           "options": {

  7906.             "alertThreshold": true

  7907.           },

  7908.           "paceLength": 10,

  7909.           "percentage": false,

  7910.           "pluginVersion": "9.2.2",

  7911.           "pointradius": 5,

  7912.           "points": false,

  7913.           "renderer": "flot",

  7914.           "seriesOverrides": [],

  7915.           "spaceLength": 10,

  7916.           "stack": false,

  7917.           "steppedLine": false,

  7918.           "targets": [

  7919.             {

  7920.               "datasource": {

  7921.                 "uid": "${DS_PROMETHEUS}"

  7922.               },

  7923.               "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])",

  7924.               "format": "time_series",

  7925.               "interval": "",

  7926.               "intervalFactor": 2,

  7927.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  7928.               "refId": "A",

  7929.               "step": 20

  7930.             }

  7931.           ],

  7932.           "thresholds": [],

  7933.           "timeRegions": [],

  7934.           "title": "Average CPU Time per Block",

  7935.           "tooltip": {

  7936.             "shared": true,

  7937.             "sort": 2,

  7938.             "value_type": "cumulative"

  7939.           },

  7940.           "type": "graph",

  7941.           "xaxis": {

  7942.             "mode": "time",

  7943.             "show": true,

  7944.             "values": []

  7945.           },

  7946.           "yaxes": [

  7947.             {

  7948.               "format": "ms",

  7949.               "logBase": 1,

  7950.               "show": true

  7951.             },

  7952.             {

  7953.               "format": "short",

  7954.               "logBase": 1,

  7955.               "show": true

  7956.             }

  7957.           ],

  7958.           "yaxis": {

  7959.             "align": false

  7960.           }

  7961.         },

  7962.         {

  7963.           "aliasColors": {},

  7964.           "bars": false,

  7965.           "dashLength": 10,

  7966.           "dashes": false,

  7967.           "datasource": {

  7968.             "type": "prometheus",

  7969.             "uid": "${DS_PROMETHEUS}"

  7970.           },

  7971.           "editable": true,

  7972.           "error": false,

  7973.           "fieldConfig": {

  7974.             "defaults": {

  7975.               "links": []

  7976.             },

  7977.             "overrides": []

  7978.           },

  7979.           "fill": 1,

  7980.           "fillGradient": 0,

  7981.           "grid": {},

  7982.           "gridPos": {

  7983.             "h": 13,

  7984.             "w": 12,

  7985.             "x": 0,

  7986.             "y": 171

  7987.           },

  7988.           "hiddenSeries": false,

  7989.           "id": 13,

  7990.           "legend": {

  7991.             "alignAsTable": true,

  7992.             "avg": false,

  7993.             "current": false,

  7994.             "max": false,

  7995.             "min": false,

  7996.             "show": true,

  7997.             "total": false,

  7998.             "values": false

  7999.           },

  8000.           "lines": true,

  8001.           "linewidth": 2,

  8002.           "links": [],

  8003.           "nullPointMode": "null",

  8004.           "options": {

  8005.             "alertThreshold": true

  8006.           },

  8007.           "paceLength": 10,

  8008.           "percentage": false,

  8009.           "pluginVersion": "9.2.2",

  8010.           "pointradius": 5,

  8011.           "points": false,

  8012.           "renderer": "flot",

  8013.           "seriesOverrides": [],

  8014.           "spaceLength": 10,

  8015.           "stack": false,

  8016.           "steppedLine": false,

  8017.           "targets": [

  8018.             {

  8019.               "datasource": {

  8020.                 "type": "prometheus",

  8021.                 "uid": "${DS_PROMETHEUS}"

  8022.               },

  8023.               "exemplar": true,

  8024.               "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  8025.               "format": "time_series",

  8026.               "interval": "",

  8027.               "intervalFactor": 2,

  8028.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  8029.               "refId": "A",

  8030.               "step": 20

  8031.             }

  8032.           ],

  8033.           "thresholds": [],

  8034.           "timeRegions": [],

  8035.           "title": "Total DB Usage by Block",

  8036.           "tooltip": {

  8037.             "shared": true,

  8038.             "sort": 2,

  8039.             "value_type": "cumulative"

  8040.           },

  8041.           "type": "graph",

  8042.           "xaxis": {

  8043.             "mode": "time",

  8044.             "show": true,

  8045.             "values": []

  8046.           },

  8047.           "yaxes": [

  8048.             {

  8049.               "$$hashKey": "object:196",

  8050.               "format": "percentunit",

  8051.               "logBase": 1,

  8052.               "min": 0,

  8053.               "show": true

  8054.             },

  8055.             {

  8056.               "$$hashKey": "object:197",

  8057.               "format": "short",

  8058.               "logBase": 1,

  8059.               "show": true

  8060.             }

  8061.           ],

  8062.           "yaxis": {

  8063.             "align": false

  8064.           }

  8065.         },

  8066.         {

  8067.           "aliasColors": {},

  8068.           "bars": false,

  8069.           "dashLength": 10,

  8070.           "dashes": false,

  8071.           "datasource": {

  8072.             "uid": "${DS_PROMETHEUS}"

  8073.           },

  8074.           "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",

  8075.           "editable": true,

  8076.           "error": false,

  8077.           "fieldConfig": {

  8078.             "defaults": {

  8079.               "links": []

  8080.             },

  8081.             "overrides": []

  8082.           },

  8083.           "fill": 1,

  8084.           "fillGradient": 0,

  8085.           "grid": {},

  8086.           "gridPos": {

  8087.             "h": 13,

  8088.             "w": 12,

  8089.             "x": 12,

  8090.             "y": 171

  8091.           },

  8092.           "hiddenSeries": false,

  8093.           "id": 27,

  8094.           "legend": {

  8095.             "alignAsTable": true,

  8096.             "avg": false,

  8097.             "current": false,

  8098.             "max": false,

  8099.             "min": false,

  8100.             "show": true,

  8101.             "total": false,

  8102.             "values": false

  8103.           },

  8104.           "lines": true,

  8105.           "linewidth": 2,

  8106.           "links": [],

  8107.           "nullPointMode": "null",

  8108.           "options": {

  8109.             "alertThreshold": true

  8110.           },

  8111.           "paceLength": 10,

  8112.           "percentage": false,

  8113.           "pluginVersion": "9.2.2",

  8114.           "pointradius": 5,

  8115.           "points": false,

  8116.           "renderer": "flot",

  8117.           "seriesOverrides": [],

  8118.           "spaceLength": 10,

  8119.           "stack": false,

  8120.           "steppedLine": false,

  8121.           "targets": [

  8122.             {

  8123.               "datasource": {

  8124.                 "uid": "${DS_PROMETHEUS}"

  8125.               },

  8126.               "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  8127.               "format": "time_series",

  8128.               "interval": "",

  8129.               "intervalFactor": 2,

  8130.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  8131.               "refId": "A",

  8132.               "step": 20

  8133.             }

  8134.           ],

  8135.           "thresholds": [],

  8136.           "timeRegions": [],

  8137.           "title": "Average Database Transaction time, by Block",

  8138.           "tooltip": {

  8139.             "shared": true,

  8140.             "sort": 2,

  8141.             "value_type": "cumulative"

  8142.           },

  8143.           "type": "graph",

  8144.           "xaxis": {

  8145.             "mode": "time",

  8146.             "show": true,

  8147.             "values": []

  8148.           },

  8149.           "yaxes": [

  8150.             {

  8151.               "format": "ms",

  8152.               "logBase": 1,

  8153.               "show": true

  8154.             },

  8155.             {

  8156.               "format": "short",

  8157.               "logBase": 1,

  8158.               "show": true

  8159.             }

  8160.           ],

  8161.           "yaxis": {

  8162.             "align": false

  8163.           }

  8164.         },

  8165.         {

  8166.           "aliasColors": {},

  8167.           "bars": false,

  8168.           "dashLength": 10,

  8169.           "dashes": false,

  8170.           "datasource": {

  8171.             "uid": "${DS_PROMETHEUS}"

  8172.           },

  8173.           "editable": true,

  8174.           "error": false,

  8175.           "fieldConfig": {

  8176.             "defaults": {

  8177.               "links": []

  8178.             },

  8179.             "overrides": []

  8180.           },

  8181.           "fill": 1,

  8182.           "fillGradient": 0,

  8183.           "grid": {},

  8184.           "gridPos": {

  8185.             "h": 13,

  8186.             "w": 12,

  8187.             "x": 0,

  8188.             "y": 184

  8189.           },

  8190.           "hiddenSeries": false,

  8191.           "id": 28,

  8192.           "legend": {

  8193.             "avg": false,

  8194.             "current": false,

  8195.             "max": false,

  8196.             "min": false,

  8197.             "show": false,

  8198.             "total": false,

  8199.             "values": false

  8200.           },

  8201.           "lines": true,

  8202.           "linewidth": 2,

  8203.           "links": [],

  8204.           "nullPointMode": "null",

  8205.           "options": {

  8206.             "alertThreshold": true

  8207.           },

  8208.           "paceLength": 10,

  8209.           "percentage": false,

  8210.           "pluginVersion": "9.2.2",

  8211.           "pointradius": 5,

  8212.           "points": false,

  8213.           "renderer": "flot",

  8214.           "seriesOverrides": [],

  8215.           "spaceLength": 10,

  8216.           "stack": false,

  8217.           "steppedLine": false,

  8218.           "targets": [

  8219.             {

  8220.               "datasource": {

  8221.                 "uid": "${DS_PROMETHEUS}"

  8222.               },

  8223.               "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  8224.               "format": "time_series",

  8225.               "interval": "",

  8226.               "intervalFactor": 2,

  8227.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  8228.               "refId": "A",

  8229.               "step": 20

  8230.             }

  8231.           ],

  8232.           "thresholds": [],

  8233.           "timeRegions": [],

  8234.           "title": "Average Transactions per Block",

  8235.           "tooltip": {

  8236.             "shared": false,

  8237.             "sort": 0,

  8238.             "value_type": "cumulative"

  8239.           },

  8240.           "type": "graph",

  8241.           "xaxis": {

  8242.             "mode": "time",

  8243.             "show": true,

  8244.             "values": []

  8245.           },

  8246.           "yaxes": [

  8247.             {

  8248.               "format": "none",

  8249.               "logBase": 1,

  8250.               "show": true

  8251.             },

  8252.             {

  8253.               "format": "short",

  8254.               "logBase": 1,

  8255.               "show": true

  8256.             }

  8257.           ],

  8258.           "yaxis": {

  8259.             "align": false

  8260.           }

  8261.         },

  8262.         {

  8263.           "aliasColors": {},

  8264.           "bars": false,

  8265.           "dashLength": 10,

  8266.           "dashes": false,

  8267.           "datasource": {

  8268.             "uid": "${DS_PROMETHEUS}"

  8269.           },

  8270.           "editable": true,

  8271.           "error": false,

  8272.           "fieldConfig": {

  8273.             "defaults": {

  8274.               "links": []

  8275.             },

  8276.             "overrides": []

  8277.           },

  8278.           "fill": 1,

  8279.           "fillGradient": 0,

  8280.           "grid": {},

  8281.           "gridPos": {

  8282.             "h": 13,

  8283.             "w": 12,

  8284.             "x": 12,

  8285.             "y": 184

  8286.           },

  8287.           "hiddenSeries": false,

  8288.           "id": 25,

  8289.           "legend": {

  8290.             "avg": false,

  8291.             "current": false,

  8292.             "max": false,

  8293.             "min": false,

  8294.             "show": false,

  8295.             "total": false,

  8296.             "values": false

  8297.           },

  8298.           "lines": true,

  8299.           "linewidth": 2,

  8300.           "links": [],

  8301.           "nullPointMode": "null",

  8302.           "options": {

  8303.             "alertThreshold": true

  8304.           },

  8305.           "paceLength": 10,

  8306.           "percentage": false,

  8307.           "pluginVersion": "9.2.2",

  8308.           "pointradius": 5,

  8309.           "points": false,

  8310.           "renderer": "flot",

  8311.           "seriesOverrides": [],

  8312.           "spaceLength": 10,

  8313.           "stack": false,

  8314.           "steppedLine": false,

  8315.           "targets": [

  8316.             {

  8317.               "datasource": {

  8318.                 "uid": "${DS_PROMETHEUS}"

  8319.               },

  8320.               "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  8321.               "format": "time_series",

  8322.               "interval": "",

  8323.               "intervalFactor": 2,

  8324.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  8325.               "refId": "A",

  8326.               "step": 20

  8327.             }

  8328.           ],

  8329.           "thresholds": [],

  8330.           "timeRegions": [],

  8331.           "title": "Average Wallclock Time per Block",

  8332.           "tooltip": {

  8333.             "shared": false,

  8334.             "sort": 0,

  8335.             "value_type": "cumulative"

  8336.           },

  8337.           "type": "graph",

  8338.           "xaxis": {

  8339.             "mode": "time",

  8340.             "show": true,

  8341.             "values": []

  8342.           },

  8343.           "yaxes": [

  8344.             {

  8345.               "$$hashKey": "object:180",

  8346.               "format": "s",

  8347.               "logBase": 1,

  8348.               "show": true

  8349.             },

  8350.             {

  8351.               "$$hashKey": "object:181",

  8352.               "format": "short",

  8353.               "logBase": 1,

  8354.               "show": true

  8355.             }

  8356.           ],

  8357.           "yaxis": {

  8358.             "align": false

  8359.           }

  8360.         },

  8361.         {

  8362.           "aliasColors": {},

  8363.           "bars": false,

  8364.           "dashLength": 10,

  8365.           "dashes": false,

  8366.           "datasource": {

  8367.             "uid": "${DS_PROMETHEUS}"

  8368.           },

  8369.           "fill": 1,

  8370.           "fillGradient": 0,

  8371.           "gridPos": {

  8372.             "h": 15,

  8373.             "w": 12,

  8374.             "x": 0,

  8375.             "y": 197

  8376.           },

  8377.           "hiddenSeries": false,

  8378.           "id": 154,

  8379.           "legend": {

  8380.             "alignAsTable": true,

  8381.             "avg": false,

  8382.             "current": false,

  8383.             "max": false,

  8384.             "min": false,

  8385.             "show": true,

  8386.             "total": false,

  8387.             "values": false

  8388.           },

  8389.           "lines": true,

  8390.           "linewidth": 1,

  8391.           "nullPointMode": "null",

  8392.           "options": {

  8393.             "alertThreshold": true

  8394.           },

  8395.           "percentage": false,

  8396.           "pluginVersion": "9.2.2",

  8397.           "pointradius": 2,

  8398.           "points": false,

  8399.           "renderer": "flot",

  8400.           "seriesOverrides": [],

  8401.           "spaceLength": 10,

  8402.           "stack": false,

  8403.           "steppedLine": false,

  8404.           "targets": [

  8405.             {

  8406.               "datasource": {

  8407.                 "uid": "${DS_PROMETHEUS}"

  8408.               },

  8409.               "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  8410.               "interval": "",

  8411.               "legendFormat": "{{job}}-{{index}} {{block_name}}",

  8412.               "refId": "A"

  8413.             }

  8414.           ],

  8415.           "thresholds": [],

  8416.           "timeRegions": [],

  8417.           "title": "Block count",

  8418.           "tooltip": {

  8419.             "shared": true,

  8420.             "sort": 2,

  8421.             "value_type": "individual"

  8422.           },

  8423.           "type": "graph",

  8424.           "xaxis": {

  8425.             "mode": "time",

  8426.             "show": true,

  8427.             "values": []

  8428.           },

  8429.           "yaxes": [

  8430.             {

  8431.               "format": "hertz",

  8432.               "logBase": 1,

  8433.               "show": true

  8434.             },

  8435.             {

  8436.               "format": "short",

  8437.               "logBase": 1,

  8438.               "show": true

  8439.             }

  8440.           ],

  8441.           "yaxis": {

  8442.             "align": false

  8443.           }

  8444.         }

  8445.       ],

  8446.       "targets": [

  8447.         {

  8448.           "datasource": {

  8449.             "type": "prometheus",

  8450.             "uid": "${DS_PROMETHEUS}"

  8451.           },

  8452.           "refId": "A"

  8453.         }

  8454.       ],

  8455.       "title": "Per-block metrics",

  8456.       "type": "row"

  8457.     },

  8458.     {

  8459.       "collapsed": true,

  8460.       "datasource": {

  8461.         "type": "prometheus",

  8462.         "uid": "${DS_PROMETHEUS}"

  8463.       },

  8464.       "gridPos": {

  8465.         "h": 1,

  8466.         "w": 24,

  8467.         "x": 0,

  8468.         "y": 36

  8469.       },

  8470.       "id": 61,

  8471.       "panels": [

  8472.         {

  8473.           "aliasColors": {},

  8474.           "bars": false,

  8475.           "dashLength": 10,

  8476.           "dashes": false,

  8477.           "datasource": {

  8478.             "uid": "${DS_PROMETHEUS}"

  8479.           },

  8480.           "decimals": 2,

  8481.           "editable": true,

  8482.           "error": false,

  8483.           "fieldConfig": {

  8484.             "defaults": {

  8485.               "links": []

  8486.             },

  8487.             "overrides": []

  8488.           },

  8489.           "fill": 0,

  8490.           "fillGradient": 0,

  8491.           "grid": {},

  8492.           "gridPos": {

  8493.             "h": 10,

  8494.             "w": 12,

  8495.             "x": 0,

  8496.             "y": 36

  8497.           },

  8498.           "hiddenSeries": false,

  8499.           "id": 1,

  8500.           "legend": {

  8501.             "alignAsTable": true,

  8502.             "avg": false,

  8503.             "current": false,

  8504.             "hideEmpty": true,

  8505.             "hideZero": false,

  8506.             "max": false,

  8507.             "min": false,

  8508.             "show": true,

  8509.             "total": false,

  8510.             "values": false

  8511.           },

  8512.           "lines": true,

  8513.           "linewidth": 2,

  8514.           "links": [],

  8515.           "nullPointMode": "null",

  8516.           "options": {

  8517.             "alertThreshold": true

  8518.           },

  8519.           "percentage": false,

  8520.           "pluginVersion": "9.0.4",

  8521.           "pointradius": 5,

  8522.           "points": false,

  8523.           "renderer": "flot",

  8524.           "seriesOverrides": [],

  8525.           "spaceLength": 10,

  8526.           "stack": false,

  8527.           "steppedLine": false,

  8528.           "targets": [

  8529.             {

  8530.               "datasource": {

  8531.                 "uid": "${DS_PROMETHEUS}"

  8532.               },

  8533.               "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  8534.               "format": "time_series",

  8535.               "intervalFactor": 2,

  8536.               "legendFormat": "{{name}} {{job}}-{{index}}",

  8537.               "refId": "A",

  8538.               "step": 20

  8539.             }

  8540.           ],

  8541.           "thresholds": [],

  8542.           "timeRegions": [],

  8543.           "title": "Cache Hit Ratio",

  8544.           "tooltip": {

  8545.             "msResolution": true,

  8546.             "shared": false,

  8547.             "sort": 0,

  8548.             "value_type": "cumulative"

  8549.           },

  8550.           "type": "graph",

  8551.           "xaxis": {

  8552.             "mode": "time",

  8553.             "show": true,

  8554.             "values": []

  8555.           },

  8556.           "yaxes": [

  8557.             {

  8558.               "format": "percentunit",

  8559.               "label": "",

  8560.               "logBase": 1,

  8561.               "max": "1",

  8562.               "min": 0,

  8563.               "show": true

  8564.             },

  8565.             {

  8566.               "format": "short",

  8567.               "logBase": 1,

  8568.               "show": false

  8569.             }

  8570.           ],

  8571.           "yaxis": {

  8572.             "align": false

  8573.           }

  8574.         },

  8575.         {

  8576.           "aliasColors": {},

  8577.           "bars": false,

  8578.           "dashLength": 10,

  8579.           "dashes": false,

  8580.           "datasource": {

  8581.             "uid": "${DS_PROMETHEUS}"

  8582.           },

  8583.           "editable": true,

  8584.           "error": false,

  8585.           "fieldConfig": {

  8586.             "defaults": {

  8587.               "links": []

  8588.             },

  8589.             "overrides": []

  8590.           },

  8591.           "fill": 1,

  8592.           "fillGradient": 0,

  8593.           "grid": {},

  8594.           "gridPos": {

  8595.             "h": 10,

  8596.             "w": 12,

  8597.             "x": 12,

  8598.             "y": 36

  8599.           },

  8600.           "hiddenSeries": false,

  8601.           "id": 8,

  8602.           "legend": {

  8603.             "alignAsTable": true,

  8604.             "avg": false,

  8605.             "current": false,

  8606.             "hideZero": false,

  8607.             "max": false,

  8608.             "min": false,

  8609.             "show": true,

  8610.             "total": false,

  8611.             "values": false

  8612.           },

  8613.           "lines": true,

  8614.           "linewidth": 2,

  8615.           "links": [],

  8616.           "nullPointMode": "connected",

  8617.           "options": {

  8618.             "alertThreshold": true

  8619.           },

  8620.           "percentage": false,

  8621.           "pluginVersion": "9.0.4",

  8622.           "pointradius": 5,

  8623.           "points": false,

  8624.           "renderer": "flot",

  8625.           "seriesOverrides": [],

  8626.           "spaceLength": 10,

  8627.           "stack": false,

  8628.           "steppedLine": false,

  8629.           "targets": [

  8630.             {

  8631.               "datasource": {

  8632.                 "uid": "${DS_PROMETHEUS}"

  8633.               },

  8634.               "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  8635.               "format": "time_series",

  8636.               "hide": false,

  8637.               "interval": "",

  8638.               "intervalFactor": 2,

  8639.               "legendFormat": "{{name}} {{job}}-{{index}}",

  8640.               "refId": "A",

  8641.               "step": 20

  8642.             }

  8643.           ],

  8644.           "thresholds": [],

  8645.           "timeRegions": [],

  8646.           "title": "Cache Size",

  8647.           "tooltip": {

  8648.             "shared": false,

  8649.             "sort": 0,

  8650.             "value_type": "cumulative"

  8651.           },

  8652.           "type": "graph",

  8653.           "xaxis": {

  8654.             "mode": "time",

  8655.             "show": true,

  8656.             "values": []

  8657.           },

  8658.           "yaxes": [

  8659.             {

  8660.               "format": "short",

  8661.               "logBase": 1,

  8662.               "min": 0,

  8663.               "show": true

  8664.             },

  8665.             {

  8666.               "format": "short",

  8667.               "logBase": 1,

  8668.               "show": true

  8669.             }

  8670.           ],

  8671.           "yaxis": {

  8672.             "align": false

  8673.           }

  8674.         },

  8675.         {

  8676.           "aliasColors": {},

  8677.           "bars": false,

  8678.           "dashLength": 10,

  8679.           "dashes": false,

  8680.           "datasource": {

  8681.             "uid": "${DS_PROMETHEUS}"

  8682.           },

  8683.           "editable": true,

  8684.           "error": false,

  8685.           "fieldConfig": {

  8686.             "defaults": {

  8687.               "links": []

  8688.             },

  8689.             "overrides": []

  8690.           },

  8691.           "fill": 1,

  8692.           "fillGradient": 0,

  8693.           "grid": {},

  8694.           "gridPos": {

  8695.             "h": 10,

  8696.             "w": 12,

  8697.             "x": 0,

  8698.             "y": 46

  8699.           },

  8700.           "hiddenSeries": false,

  8701.           "id": 38,

  8702.           "legend": {

  8703.             "alignAsTable": true,

  8704.             "avg": false,

  8705.             "current": false,

  8706.             "hideZero": false,

  8707.             "max": false,

  8708.             "min": false,

  8709.             "show": true,

  8710.             "total": false,

  8711.             "values": false

  8712.           },

  8713.           "lines": true,

  8714.           "linewidth": 2,

  8715.           "links": [],

  8716.           "nullPointMode": "connected",

  8717.           "options": {

  8718.             "alertThreshold": true

  8719.           },

  8720.           "percentage": false,

  8721.           "pluginVersion": "9.0.4",

  8722.           "pointradius": 5,

  8723.           "points": false,

  8724.           "renderer": "flot",

  8725.           "seriesOverrides": [],

  8726.           "spaceLength": 10,

  8727.           "stack": false,

  8728.           "steppedLine": false,

  8729.           "targets": [

  8730.             {

  8731.               "datasource": {

  8732.                 "uid": "${DS_PROMETHEUS}"

  8733.               },

  8734.               "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  8735.               "format": "time_series",

  8736.               "interval": "",

  8737.               "intervalFactor": 2,

  8738.               "legendFormat": "{{name}} {{job}}-{{index}}",

  8739.               "refId": "A",

  8740.               "step": 20

  8741.             }

  8742.           ],

  8743.           "thresholds": [],

  8744.           "timeRegions": [],

  8745.           "title": "Cache request rate",

  8746.           "tooltip": {

  8747.             "shared": false,

  8748.             "sort": 0,

  8749.             "value_type": "cumulative"

  8750.           },

  8751.           "type": "graph",

  8752.           "xaxis": {

  8753.             "mode": "time",

  8754.             "show": true,

  8755.             "values": []

  8756.           },

  8757.           "yaxes": [

  8758.             {

  8759.               "format": "rps",

  8760.               "logBase": 1,

  8761.               "min": 0,

  8762.               "show": true

  8763.             },

  8764.             {

  8765.               "format": "short",

  8766.               "logBase": 1,

  8767.               "show": true

  8768.             }

  8769.           ],

  8770.           "yaxis": {

  8771.             "align": false

  8772.           }

  8773.         },

  8774.         {

  8775.           "aliasColors": {},

  8776.           "bars": false,

  8777.           "dashLength": 10,

  8778.           "dashes": false,

  8779.           "datasource": {

  8780.             "uid": "${DS_PROMETHEUS}"

  8781.           },

  8782.           "fieldConfig": {

  8783.             "defaults": {

  8784.               "links": []

  8785.             },

  8786.             "overrides": []

  8787.           },

  8788.           "fill": 1,

  8789.           "fillGradient": 0,

  8790.           "gridPos": {

  8791.             "h": 10,

  8792.             "w": 12,

  8793.             "x": 12,

  8794.             "y": 46

  8795.           },

  8796.           "hiddenSeries": false,

  8797.           "id": 39,

  8798.           "legend": {

  8799.             "alignAsTable": true,

  8800.             "avg": false,

  8801.             "current": false,

  8802.             "hideEmpty": false,

  8803.             "hideZero": false,

  8804.             "max": true,

  8805.             "min": false,

  8806.             "rightSide": false,

  8807.             "show": true,

  8808.             "sort": "max",

  8809.             "sortDesc": true,

  8810.             "total": false,

  8811.             "values": true

  8812.           },

  8813.           "lines": true,

  8814.           "linewidth": 1,

  8815.           "links": [],

  8816.           "nullPointMode": "null",

  8817.           "options": {

  8818.             "alertThreshold": true

  8819.           },

  8820.           "percentage": false,

  8821.           "pluginVersion": "9.0.4",

  8822.           "pointradius": 5,

  8823.           "points": false,

  8824.           "renderer": "flot",

  8825.           "seriesOverrides": [],

  8826.           "spaceLength": 10,

  8827.           "stack": false,

  8828.           "steppedLine": false,

  8829.           "targets": [

  8830.             {

  8831.               "datasource": {

  8832.                 "uid": "${DS_PROMETHEUS}"

  8833.               },

  8834.               "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",

  8835.               "format": "time_series",

  8836.               "interval": "",

  8837.               "intervalFactor": 2,

  8838.               "legendFormat": "{{name}} {{job}}-{{index}}",

  8839.               "refId": "A",

  8840.               "step": 20

  8841.             }

  8842.           ],

  8843.           "thresholds": [],

  8844.           "timeRegions": [],

  8845.           "title": "Top 10 cache misses",

  8846.           "tooltip": {

  8847.             "shared": false,

  8848.             "sort": 0,

  8849.             "value_type": "individual"

  8850.           },

  8851.           "type": "graph",

  8852.           "xaxis": {

  8853.             "mode": "time",

  8854.             "show": true,

  8855.             "values": []

  8856.           },

  8857.           "yaxes": [

  8858.             {

  8859.               "$$hashKey": "object:101",

  8860.               "format": "rps",

  8861.               "logBase": 1,

  8862.               "show": true

  8863.             },

  8864.             {

  8865.               "$$hashKey": "object:102",

  8866.               "format": "short",

  8867.               "logBase": 1,

  8868.               "show": true

  8869.             }

  8870.           ],

  8871.           "yaxis": {

  8872.             "align": false

  8873.           }

  8874.         },

  8875.         {

  8876.           "aliasColors": {},

  8877.           "bars": false,

  8878.           "dashLength": 10,

  8879.           "dashes": false,

  8880.           "datasource": {

  8881.             "uid": "${DS_PROMETHEUS}"

  8882.           },

  8883.           "fieldConfig": {

  8884.             "defaults": {

  8885.               "links": []

  8886.             },

  8887.             "overrides": []

  8888.           },

  8889.           "fill": 1,

  8890.           "fillGradient": 0,

  8891.           "gridPos": {

  8892.             "h": 9,

  8893.             "w": 12,

  8894.             "x": 0,

  8895.             "y": 56

  8896.           },

  8897.           "hiddenSeries": false,

  8898.           "id": 65,

  8899.           "legend": {

  8900.             "alignAsTable": true,

  8901.             "avg": false,

  8902.             "current": false,

  8903.             "max": false,

  8904.             "min": false,

  8905.             "show": true,

  8906.             "total": false,

  8907.             "values": false

  8908.           },

  8909.           "lines": true,

  8910.           "linewidth": 1,

  8911.           "links": [],

  8912.           "nullPointMode": "null",

  8913.           "options": {

  8914.             "alertThreshold": true

  8915.           },

  8916.           "percentage": false,

  8917.           "pluginVersion": "9.0.4",

  8918.           "pointradius": 5,

  8919.           "points": false,

  8920.           "renderer": "flot",

  8921.           "seriesOverrides": [],

  8922.           "spaceLength": 10,

  8923.           "stack": false,

  8924.           "steppedLine": false,

  8925.           "targets": [

  8926.             {

  8927.               "datasource": {

  8928.                 "uid": "${DS_PROMETHEUS}"

  8929.               },

  8930.               "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  8931.               "format": "time_series",

  8932.               "interval": "",

  8933.               "intervalFactor": 1,

  8934.               "legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",

  8935.               "refId": "A"

  8936.             }

  8937.           ],

  8938.           "thresholds": [],

  8939.           "timeRegions": [],

  8940.           "title": "Cache eviction rate",

  8941.           "tooltip": {

  8942.             "shared": false,

  8943.             "sort": 0,

  8944.             "value_type": "individual"

  8945.           },

  8946.           "type": "graph",

  8947.           "xaxis": {

  8948.             "mode": "time",

  8949.             "show": true,

  8950.             "values": []

  8951.           },

  8952.           "yaxes": [

  8953.             {

  8954.               "format": "hertz",

  8955.               "label": "entries / second",

  8956.               "logBase": 1,

  8957.               "show": true

  8958.             },

  8959.             {

  8960.               "format": "short",

  8961.               "logBase": 1,

  8962.               "show": true

  8963.             }

  8964.           ],

  8965.           "yaxis": {

  8966.             "align": false

  8967.           }

  8968.         }

  8969.       ],

  8970.       "targets": [

  8971.         {

  8972.           "datasource": {

  8973.             "type": "prometheus",

  8974.             "uid": "${DS_PROMETHEUS}"

  8975.           },

  8976.           "refId": "A"

  8977.         }

  8978.       ],

  8979.       "title": "Caches",

  8980.       "type": "row"

  8981.     },

  8982.     {

  8983.       "collapsed": true,

  8984.       "datasource": {

  8985.         "type": "prometheus",

  8986.         "uid": "${DS_PROMETHEUS}"

  8987.       },

  8988.       "gridPos": {

  8989.         "h": 1,

  8990.         "w": 24,

  8991.         "x": 0,

  8992.         "y": 37

  8993.       },

  8994.       "id": 148,

  8995.       "panels": [

  8996.         {

  8997.           "aliasColors": {},

  8998.           "bars": false,

  8999.           "dashLength": 10,

  9000.           "dashes": false,

  9001.           "datasource": {

  9002.             "uid": "${DS_PROMETHEUS}"

  9003.           },

  9004.           "fieldConfig": {

  9005.             "defaults": {

  9006.               "custom": {},

  9007.               "links": []

  9008.             },

  9009.             "overrides": []

  9010.           },

  9011.           "fill": 1,

  9012.           "fillGradient": 0,

  9013.           "gridPos": {

  9014.             "h": 8,

  9015.             "w": 12,

  9016.             "x": 0,

  9017.             "y": 29

  9018.           },

  9019.           "hiddenSeries": false,

  9020.           "id": 146,

  9021.           "legend": {

  9022.             "avg": false,

  9023.             "current": false,

  9024.             "max": false,

  9025.             "min": false,

  9026.             "show": true,

  9027.             "total": false,

  9028.             "values": false

  9029.           },

  9030.           "lines": true,

  9031.           "linewidth": 1,

  9032.           "nullPointMode": "null",

  9033.           "options": {

  9034.             "alertThreshold": true

  9035.           },

  9036.           "percentage": false,

  9037.           "pluginVersion": "7.3.7",

  9038.           "pointradius": 2,

  9039.           "points": false,

  9040.           "renderer": "flot",

  9041.           "seriesOverrides": [],

  9042.           "spaceLength": 10,

  9043.           "stack": false,

  9044.           "steppedLine": false,

  9045.           "targets": [

  9046.             {

  9047.               "datasource": {

  9048.                 "uid": "${DS_PROMETHEUS}"

  9049.               },

  9050.               "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  9051.               "interval": "",

  9052.               "legendFormat": "{{name}} {{job}}-{{index}}",

  9053.               "refId": "A"

  9054.             }

  9055.           ],

  9056.           "thresholds": [],

  9057.           "timeRegions": [],

  9058.           "title": "Response cache size",

  9059.           "tooltip": {

  9060.             "shared": false,

  9061.             "sort": 0,

  9062.             "value_type": "individual"

  9063.           },

  9064.           "type": "graph",

  9065.           "xaxis": {

  9066.             "mode": "time",

  9067.             "show": true,

  9068.             "values": []

  9069.           },

  9070.           "yaxes": [

  9071.             {

  9072.               "format": "short",

  9073.               "logBase": 1,

  9074.               "show": true

  9075.             },

  9076.             {

  9077.               "format": "short",

  9078.               "logBase": 1,

  9079.               "show": true

  9080.             }

  9081.           ],

  9082.           "yaxis": {

  9083.             "align": false

  9084.           }

  9085.         },

  9086.         {

  9087.           "aliasColors": {},

  9088.           "bars": false,

  9089.           "dashLength": 10,

  9090.           "dashes": false,

  9091.           "datasource": {

  9092.             "uid": "${DS_PROMETHEUS}"

  9093.           },

  9094.           "fieldConfig": {

  9095.             "defaults": {

  9096.               "custom": {},

  9097.               "links": []

  9098.             },

  9099.             "overrides": []

  9100.           },

  9101.           "fill": 1,

  9102.           "fillGradient": 0,

  9103.           "gridPos": {

  9104.             "h": 8,

  9105.             "w": 12,

  9106.             "x": 12,

  9107.             "y": 29

  9108.           },

  9109.           "hiddenSeries": false,

  9110.           "id": 150,

  9111.           "legend": {

  9112.             "avg": false,

  9113.             "current": false,

  9114.             "max": false,

  9115.             "min": false,

  9116.             "show": true,

  9117.             "total": false,

  9118.             "values": false

  9119.           },

  9120.           "lines": true,

  9121.           "linewidth": 1,

  9122.           "nullPointMode": "null",

  9123.           "options": {

  9124.             "alertThreshold": true

  9125.           },

  9126.           "percentage": false,

  9127.           "pluginVersion": "7.3.7",

  9128.           "pointradius": 2,

  9129.           "points": false,

  9130.           "renderer": "flot",

  9131.           "seriesOverrides": [],

  9132.           "spaceLength": 10,

  9133.           "stack": false,

  9134.           "steppedLine": false,

  9135.           "targets": [

  9136.             {

  9137.               "datasource": {

  9138.                 "uid": "${DS_PROMETHEUS}"

  9139.               },

  9140.               "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",

  9141.               "interval": "",

  9142.               "legendFormat": "{{name}} {{job}}-{{index}}",

  9143.               "refId": "A"

  9144.             },

  9145.             {

  9146.               "datasource": {

  9147.                 "uid": "${DS_PROMETHEUS}"

  9148.               },

  9149.               "expr": "",

  9150.               "interval": "",

  9151.               "legendFormat": "",

  9152.               "refId": "B"

  9153.             }

  9154.           ],

  9155.           "thresholds": [],

  9156.           "timeRegions": [],

  9157.           "title": "Response cache hit rate",

  9158.           "tooltip": {

  9159.             "shared": false,

  9160.             "sort": 0,

  9161.             "value_type": "individual"

  9162.           },

  9163.           "type": "graph",

  9164.           "xaxis": {

  9165.             "mode": "time",

  9166.             "show": true,

  9167.             "values": []

  9168.           },

  9169.           "yaxes": [

  9170.             {

  9171.               "format": "percentunit",

  9172.               "logBase": 1,

  9173.               "max": "1",

  9174.               "min": "0",

  9175.               "show": true

  9176.             },

  9177.             {

  9178.               "format": "short",

  9179.               "logBase": 1,

  9180.               "show": true

  9181.             }

  9182.           ],

  9183.           "yaxis": {

  9184.             "align": false

  9185.           }

  9186.         }

  9187.       ],

  9188.       "targets": [

  9189.         {

  9190.           "datasource": {

  9191.             "type": "prometheus",

  9192.             "uid": "${DS_PROMETHEUS}"

  9193.           },

  9194.           "refId": "A"

  9195.         }

  9196.       ],

  9197.       "title": "Response caches",

  9198.       "type": "row"

  9199.     },

  9200.     {

  9201.       "collapsed": true,

  9202.       "datasource": {

  9203.         "type": "prometheus",

  9204.         "uid": "${DS_PROMETHEUS}"

  9205.       },

  9206.       "gridPos": {

  9207.         "h": 1,

  9208.         "w": 24,

  9209.         "x": 0,

  9210.         "y": 38

  9211.       },

  9212.       "id": 62,

  9213.       "panels": [

  9214.         {

  9215.           "aliasColors": {},

  9216.           "bars": false,

  9217.           "dashLength": 10,

  9218.           "dashes": false,

  9219.           "datasource": {

  9220.             "uid": "${DS_PROMETHEUS}"

  9221.           },

  9222.           "fieldConfig": {

  9223.             "defaults": {

  9224.               "custom": {},

  9225.               "links": []

  9226.             },

  9227.             "overrides": []

  9228.           },

  9229.           "fill": 1,

  9230.           "fillGradient": 0,

  9231.           "gridPos": {

  9232.             "h": 9,

  9233.             "w": 12,

  9234.             "x": 0,

  9235.             "y": 30

  9236.           },

  9237.           "hiddenSeries": false,

  9238.           "id": 91,

  9239.           "legend": {

  9240.             "avg": false,

  9241.             "current": false,

  9242.             "max": false,

  9243.             "min": false,

  9244.             "show": true,

  9245.             "total": false,

  9246.             "values": false

  9247.           },

  9248.           "lines": true,

  9249.           "linewidth": 1,

  9250.           "links": [],

  9251.           "nullPointMode": "null",

  9252.           "options": {

  9253.             "alertThreshold": true

  9254.           },

  9255.           "percentage": false,

  9256.           "pluginVersion": "7.3.7",

  9257.           "pointradius": 5,

  9258.           "points": false,

  9259.           "renderer": "flot",

  9260.           "seriesOverrides": [],

  9261.           "spaceLength": 10,

  9262.           "stack": true,

  9263.           "steppedLine": false,

  9264.           "targets": [

  9265.             {

  9266.               "datasource": {

  9267.                 "uid": "${DS_PROMETHEUS}"

  9268.               },

  9269.               "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",

  9270.               "format": "time_series",

  9271.               "instant": false,

  9272.               "intervalFactor": 1,

  9273.               "legendFormat": "{{job}}-{{index}} gen {{gen}}",

  9274.               "refId": "A"

  9275.             }

  9276.           ],

  9277.           "thresholds": [],

  9278.           "timeRegions": [],

  9279.           "title": "Total GC time by bucket (10m smoothing)",

  9280.           "tooltip": {

  9281.             "shared": true,

  9282.             "sort": 0,

  9283.             "value_type": "individual"

  9284.           },

  9285.           "type": "graph",

  9286.           "xaxis": {

  9287.             "mode": "time",

  9288.             "show": true,

  9289.             "values": []

  9290.           },

  9291.           "yaxes": [

  9292.             {

  9293.               "format": "percentunit",

  9294.               "logBase": 1,

  9295.               "min": "0",

  9296.               "show": true

  9297.             },

  9298.             {

  9299.               "format": "short",

  9300.               "logBase": 1,

  9301.               "show": true

  9302.             }

  9303.           ],

  9304.           "yaxis": {

  9305.             "align": false

  9306.           }

  9307.         },

  9308.         {

  9309.           "aliasColors": {},

  9310.           "bars": false,

  9311.           "dashLength": 10,

  9312.           "dashes": false,

  9313.           "datasource": {

  9314.             "uid": "${DS_PROMETHEUS}"

  9315.           },

  9316.           "decimals": 3,

  9317.           "editable": true,

  9318.           "error": false,

  9319.           "fieldConfig": {

  9320.             "defaults": {

  9321.               "custom": {},

  9322.               "links": []

  9323.             },

  9324.             "overrides": []

  9325.           },

  9326.           "fill": 1,

  9327.           "fillGradient": 0,

  9328.           "grid": {},

  9329.           "gridPos": {

  9330.             "h": 9,

  9331.             "w": 12,

  9332.             "x": 12,

  9333.             "y": 30

  9334.           },

  9335.           "hiddenSeries": false,

  9336.           "id": 21,

  9337.           "legend": {

  9338.             "alignAsTable": true,

  9339.             "avg": false,

  9340.             "current": false,

  9341.             "max": false,

  9342.             "min": false,

  9343.             "show": true,

  9344.             "total": false,

  9345.             "values": false

  9346.           },

  9347.           "lines": true,

  9348.           "linewidth": 2,

  9349.           "links": [],

  9350.           "nullPointMode": "null as zero",

  9351.           "options": {

  9352.             "alertThreshold": true

  9353.           },

  9354.           "percentage": false,

  9355.           "pluginVersion": "7.3.7",

  9356.           "pointradius": 5,

  9357.           "points": false,

  9358.           "renderer": "flot",

  9359.           "seriesOverrides": [],

  9360.           "spaceLength": 10,

  9361.           "stack": false,

  9362.           "steppedLine": false,

  9363.           "targets": [

  9364.             {

  9365.               "datasource": {

  9366.                 "uid": "${DS_PROMETHEUS}"

  9367.               },

  9368.               "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",

  9369.               "format": "time_series",

  9370.               "intervalFactor": 2,

  9371.               "legendFormat": "{{job}} {{index}} gen {{gen}} ",

  9372.               "refId": "A",

  9373.               "step": 20,

  9374.               "target": ""

  9375.             }

  9376.           ],

  9377.           "thresholds": [],

  9378.           "timeRegions": [],

  9379.           "title": "Average GC Time Per Collection",

  9380.           "tooltip": {

  9381.             "shared": false,

  9382.             "sort": 0,

  9383.             "value_type": "cumulative"

  9384.           },

  9385.           "type": "graph",

  9386.           "xaxis": {

  9387.             "mode": "time",

  9388.             "show": true,

  9389.             "values": []

  9390.           },

  9391.           "yaxes": [

  9392.             {

  9393.               "format": "s",

  9394.               "logBase": 1,

  9395.               "show": true

  9396.             },

  9397.             {

  9398.               "format": "short",

  9399.               "logBase": 1,

  9400.               "show": true

  9401.             }

  9402.           ],

  9403.           "yaxis": {

  9404.             "align": false

  9405.           }

  9406.         },

  9407.         {

  9408.           "aliasColors": {},

  9409.           "bars": false,

  9410.           "dashLength": 10,

  9411.           "dashes": false,

  9412.           "datasource": {

  9413.             "uid": "${DS_PROMETHEUS}"

  9414.           },

  9415.           "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",

  9416.           "fieldConfig": {

  9417.             "defaults": {

  9418.               "custom": {},

  9419.               "links": []

  9420.             },

  9421.             "overrides": []

  9422.           },

  9423.           "fill": 1,

  9424.           "fillGradient": 0,

  9425.           "gridPos": {

  9426.             "h": 9,

  9427.             "w": 12,

  9428.             "x": 0,

  9429.             "y": 39

  9430.           },

  9431.           "hiddenSeries": false,

  9432.           "id": 89,

  9433.           "legend": {

  9434.             "avg": false,

  9435.             "current": false,

  9436.             "hideEmpty": true,

  9437.             "hideZero": false,

  9438.             "max": false,

  9439.             "min": false,

  9440.             "show": true,

  9441.             "total": false,

  9442.             "values": false

  9443.           },

  9444.           "lines": true,

  9445.           "linewidth": 1,

  9446.           "links": [],

  9447.           "nullPointMode": "null",

  9448.           "options": {

  9449.             "alertThreshold": true

  9450.           },

  9451.           "percentage": false,

  9452.           "pluginVersion": "7.3.7",

  9453.           "pointradius": 5,

  9454.           "points": false,

  9455.           "renderer": "flot",

  9456.           "seriesOverrides": [

  9457.             {

  9458.               "alias": "/gen 0$/",

  9459.               "yaxis": 2

  9460.             }

  9461.           ],

  9462.           "spaceLength": 10,

  9463.           "stack": false,

  9464.           "steppedLine": false,

  9465.           "targets": [

  9466.             {

  9467.               "datasource": {

  9468.                 "uid": "${DS_PROMETHEUS}"

  9469.               },

  9470.               "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",

  9471.               "format": "time_series",

  9472.               "intervalFactor": 1,

  9473.               "legendFormat": "{{job}}-{{index}} gen {{gen}}",

  9474.               "refId": "A"

  9475.             }

  9476.           ],

  9477.           "thresholds": [],

  9478.           "timeRegions": [],

  9479.           "title": "Allocation counts",

  9480.           "tooltip": {

  9481.             "shared": false,

  9482.             "sort": 0,

  9483.             "value_type": "individual"

  9484.           },

  9485.           "type": "graph",

  9486.           "xaxis": {

  9487.             "mode": "time",

  9488.             "show": true,

  9489.             "values": []

  9490.           },

  9491.           "yaxes": [

  9492.             {

  9493.               "format": "short",

  9494.               "label": "Gen N-1 GCs since last Gen N GC",

  9495.               "logBase": 1,

  9496.               "show": true

  9497.             },

  9498.             {

  9499.               "format": "short",

  9500.               "label": "Objects since last Gen 0 GC",

  9501.               "logBase": 1,

  9502.               "show": true

  9503.             }

  9504.           ],

  9505.           "yaxis": {

  9506.             "align": false

  9507.           }

  9508.         },

  9509.         {

  9510.           "aliasColors": {},

  9511.           "bars": false,

  9512.           "dashLength": 10,

  9513.           "dashes": false,

  9514.           "datasource": {

  9515.             "uid": "${DS_PROMETHEUS}"

  9516.           },

  9517.           "fieldConfig": {

  9518.             "defaults": {

  9519.               "custom": {},

  9520.               "links": []

  9521.             },

  9522.             "overrides": []

  9523.           },

  9524.           "fill": 1,

  9525.           "fillGradient": 0,

  9526.           "gridPos": {

  9527.             "h": 9,

  9528.             "w": 12,

  9529.             "x": 12,

  9530.             "y": 39

  9531.           },

  9532.           "hiddenSeries": false,

  9533.           "id": 93,

  9534.           "legend": {

  9535.             "avg": false,

  9536.             "current": false,

  9537.             "max": false,

  9538.             "min": false,

  9539.             "show": true,

  9540.             "total": false,

  9541.             "values": false

  9542.           },

  9543.           "lines": true,

  9544.           "linewidth": 1,

  9545.           "links": [],

  9546.           "nullPointMode": "connected",

  9547.           "options": {

  9548.             "alertThreshold": true

  9549.           },

  9550.           "percentage": false,

  9551.           "pluginVersion": "7.3.7",

  9552.           "pointradius": 5,

  9553.           "points": false,

  9554.           "renderer": "flot",

  9555.           "seriesOverrides": [],

  9556.           "spaceLength": 10,

  9557.           "stack": false,

  9558.           "steppedLine": false,

  9559.           "targets": [

  9560.             {

  9561.               "datasource": {

  9562.                 "uid": "${DS_PROMETHEUS}"

  9563.               },

  9564.               "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  9565.               "format": "time_series",

  9566.               "intervalFactor": 1,

  9567.               "legendFormat": "{{job}}-{{index}} gen {{gen}}",

  9568.               "refId": "A"

  9569.             }

  9570.           ],

  9571.           "thresholds": [],

  9572.           "timeRegions": [],

  9573.           "title": "Object counts per collection",

  9574.           "tooltip": {

  9575.             "shared": true,

  9576.             "sort": 0,

  9577.             "value_type": "individual"

  9578.           },

  9579.           "type": "graph",

  9580.           "xaxis": {

  9581.             "mode": "time",

  9582.             "show": true,

  9583.             "values": []

  9584.           },

  9585.           "yaxes": [

  9586.             {

  9587.               "format": "short",

  9588.               "logBase": 1,

  9589.               "show": true

  9590.             },

  9591.             {

  9592.               "format": "short",

  9593.               "logBase": 1,

  9594.               "show": true

  9595.             }

  9596.           ],

  9597.           "yaxis": {

  9598.             "align": false

  9599.           }

  9600.         },

  9601.         {

  9602.           "aliasColors": {},

  9603.           "bars": false,

  9604.           "dashLength": 10,

  9605.           "dashes": false,

  9606.           "datasource": {

  9607.             "uid": "${DS_PROMETHEUS}"

  9608.           },

  9609.           "fieldConfig": {

  9610.             "defaults": {

  9611.               "custom": {},

  9612.               "links": []

  9613.             },

  9614.             "overrides": []

  9615.           },

  9616.           "fill": 1,

  9617.           "fillGradient": 0,

  9618.           "gridPos": {

  9619.             "h": 9,

  9620.             "w": 12,

  9621.             "x": 0,

  9622.             "y": 48

  9623.           },

  9624.           "hiddenSeries": false,

  9625.           "id": 95,

  9626.           "legend": {

  9627.             "avg": false,

  9628.             "current": false,

  9629.             "max": false,

  9630.             "min": false,

  9631.             "show": true,

  9632.             "total": false,

  9633.             "values": false

  9634.           },

  9635.           "lines": true,

  9636.           "linewidth": 1,

  9637.           "links": [],

  9638.           "nullPointMode": "null",

  9639.           "options": {

  9640.             "alertThreshold": true

  9641.           },

  9642.           "percentage": false,

  9643.           "pluginVersion": "7.3.7",

  9644.           "pointradius": 5,

  9645.           "points": false,

  9646.           "renderer": "flot",

  9647.           "seriesOverrides": [],

  9648.           "spaceLength": 10,

  9649.           "stack": false,

  9650.           "steppedLine": false,

  9651.           "targets": [

  9652.             {

  9653.               "datasource": {

  9654.                 "uid": "${DS_PROMETHEUS}"

  9655.               },

  9656.               "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  9657.               "format": "time_series",

  9658.               "intervalFactor": 1,

  9659.               "legendFormat": "{{job}}-{{index}} gen {{gen}}",

  9660.               "refId": "A"

  9661.             }

  9662.           ],

  9663.           "thresholds": [],

  9664.           "timeRegions": [],

  9665.           "title": "GC frequency",

  9666.           "tooltip": {

  9667.             "shared": true,

  9668.             "sort": 0,

  9669.             "value_type": "individual"

  9670.           },

  9671.           "type": "graph",

  9672.           "xaxis": {

  9673.             "mode": "time",

  9674.             "show": true,

  9675.             "values": []

  9676.           },

  9677.           "yaxes": [

  9678.             {

  9679.               "format": "hertz",

  9680.               "logBase": 1,

  9681.               "show": true

  9682.             },

  9683.             {

  9684.               "format": "short",

  9685.               "logBase": 1,

  9686.               "show": true

  9687.             }

  9688.           ],

  9689.           "yaxis": {

  9690.             "align": false

  9691.           }

  9692.         },

  9693.         {

  9694.           "cards": {

  9695.             "cardPadding": 0

  9696.           },

  9697.           "color": {

  9698.             "cardColor": "#b4ff00",

  9699.             "colorScale": "sqrt",

  9700.             "colorScheme": "interpolateSpectral",

  9701.             "exponent": 0.5,

  9702.             "min": 0,

  9703.             "mode": "spectrum"

  9704.           },

  9705.           "dataFormat": "tsbuckets",

  9706.           "datasource": {

  9707.             "type": "prometheus",

  9708.             "uid": "${DS_PROMETHEUS}"

  9709.           },

  9710.           "fieldConfig": {

  9711.             "defaults": {

  9712.               "custom": {}

  9713.             },

  9714.             "overrides": []

  9715.           },

  9716.           "gridPos": {

  9717.             "h": 9,

  9718.             "w": 12,

  9719.             "x": 12,

  9720.             "y": 48

  9721.           },

  9722.           "heatmap": {},

  9723.           "hideZeroBuckets": true,

  9724.           "highlightCards": true,

  9725.           "id": 87,

  9726.           "legend": {

  9727.             "show": true

  9728.           },

  9729.           "links": [],

  9730.           "reverseYBuckets": false,

  9731.           "targets": [

  9732.             {

  9733.               "datasource": {

  9734.                 "type": "prometheus",

  9735.                 "uid": "${DS_PROMETHEUS}"

  9736.               },

  9737.               "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",

  9738.               "format": "heatmap",

  9739.               "intervalFactor": 1,

  9740.               "legendFormat": "{{le}}",

  9741.               "refId": "A"

  9742.             }

  9743.           ],

  9744.           "title": "GC durations",

  9745.           "tooltip": {

  9746.             "show": true,

  9747.             "showHistogram": false

  9748.           },

  9749.           "type": "heatmap",

  9750.           "xAxis": {

  9751.             "show": true

  9752.           },

  9753.           "yAxis": {

  9754.             "format": "s",

  9755.             "logBase": 1,

  9756.             "show": true

  9757.           },

  9758.           "yBucketBound": "auto"

  9759.         }

  9760.       ],

  9761.       "targets": [

  9762.         {

  9763.           "datasource": {

  9764.             "type": "prometheus",

  9765.             "uid": "${DS_PROMETHEUS}"

  9766.           },

  9767.           "refId": "A"

  9768.         }

  9769.       ],

  9770.       "title": "GC",

  9771.       "type": "row"

  9772.     },

  9773.     {

  9774.       "collapsed": true,

  9775.       "datasource": {

  9776.         "type": "prometheus",

  9777.         "uid": "${DS_PROMETHEUS}"

  9778.       },

  9779.       "gridPos": {

  9780.         "h": 1,

  9781.         "w": 24,

  9782.         "x": 0,

  9783.         "y": 39

  9784.       },

  9785.       "id": 63,

  9786.       "panels": [

  9787.         {

  9788.           "aliasColors": {},

  9789.           "bars": false,

  9790.           "dashLength": 10,

  9791.           "dashes": false,

  9792.           "datasource": {

  9793.             "type": "prometheus",

  9794.             "uid": "${DS_PROMETHEUS}"

  9795.           },

  9796.           "fieldConfig": {

  9797.             "defaults": {

  9798.               "links": []

  9799.             },

  9800.             "overrides": []

  9801.           },

  9802.           "fill": 1,

  9803.           "fillGradient": 0,

  9804.           "gridPos": {

  9805.             "h": 7,

  9806.             "w": 12,

  9807.             "x": 0,

  9808.             "y": 162

  9809.           },

  9810.           "hiddenSeries": false,

  9811.           "id": 43,

  9812.           "legend": {

  9813.             "avg": false,

  9814.             "current": false,

  9815.             "max": false,

  9816.             "min": false,

  9817.             "show": true,

  9818.             "total": false,

  9819.             "values": false

  9820.           },

  9821.           "lines": true,

  9822.           "linewidth": 1,

  9823.           "links": [],

  9824.           "nullPointMode": "null",

  9825.           "options": {

  9826.             "alertThreshold": true

  9827.           },

  9828.           "paceLength": 10,

  9829.           "percentage": false,

  9830.           "pluginVersion": "9.2.2",

  9831.           "pointradius": 5,

  9832.           "points": false,

  9833.           "renderer": "flot",

  9834.           "seriesOverrides": [],

  9835.           "spaceLength": 10,

  9836.           "stack": false,

  9837.           "steppedLine": false,

  9838.           "targets": [

  9839.             {

  9840.               "datasource": {

  9841.                 "uid": "${DS_PROMETHEUS}"

  9842.               },

  9843.               "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",

  9844.               "format": "time_series",

  9845.               "intervalFactor": 2,

  9846.               "legendFormat": "{{job}}-{{index}} {{command}}",

  9847.               "refId": "A",

  9848.               "step": 20

  9849.             }

  9850.           ],

  9851.           "thresholds": [],

  9852.           "timeRegions": [],

  9853.           "title": "Rate of outgoing commands",

  9854.           "tooltip": {

  9855.             "shared": false,

  9856.             "sort": 0,

  9857.             "value_type": "individual"

  9858.           },

  9859.           "type": "graph",

  9860.           "xaxis": {

  9861.             "mode": "time",

  9862.             "show": true,

  9863.             "values": []

  9864.           },

  9865.           "yaxes": [

  9866.             {

  9867.               "$$hashKey": "object:89",

  9868.               "format": "hertz",

  9869.               "logBase": 1,

  9870.               "show": true

  9871.             },

  9872.             {

  9873.               "$$hashKey": "object:90",

  9874.               "format": "short",

  9875.               "logBase": 1,

  9876.               "show": true

  9877.             }

  9878.           ],

  9879.           "yaxis": {

  9880.             "align": false

  9881.           }

  9882.         },

  9883.         {

  9884.           "datasource": {

  9885.             "type": "prometheus",

  9886.             "uid": "${DS_PROMETHEUS}"

  9887.           },

  9888.           "fieldConfig": {

  9889.             "defaults": {

  9890.               "color": {

  9891.                 "mode": "palette-classic"

  9892.               },

  9893.               "custom": {

  9894.                 "axisCenteredZero": false,

  9895.                 "axisColorMode": "text",

  9896.                 "axisLabel": "",

  9897.                 "axisPlacement": "auto",

  9898.                 "barAlignment": 0,

  9899.                 "drawStyle": "line",

  9900.                 "fillOpacity": 10,

  9901.                 "gradientMode": "none",

  9902.                 "hideFrom": {

  9903.                   "legend": false,

  9904.                   "tooltip": false,

  9905.                   "viz": false

  9906.                 },

  9907.                 "lineInterpolation": "linear",

  9908.                 "lineWidth": 1,

  9909.                 "pointSize": 5,

  9910.                 "scaleDistribution": {

  9911.                   "type": "linear"

  9912.                 },

  9913.                 "showPoints": "never",

  9914.                 "spanNulls": true,

  9915.                 "stacking": {

  9916.                   "group": "A",

  9917.                   "mode": "none"

  9918.                 },

  9919.                 "thresholdsStyle": {

  9920.                   "mode": "off"

  9921.                 }

  9922.               },

  9923.               "links": [],

  9924.               "mappings": [],

  9925.               "thresholds": {

  9926.                 "mode": "absolute",

  9927.                 "steps": [

  9928.                   {

  9929.                     "color": "green"

  9930.                   },

  9931.                   {

  9932.                     "color": "red",

  9933.                     "value": 80

  9934.                   }

  9935.                 ]

  9936.               },

  9937.               "unit": "hertz"

  9938.             },

  9939.             "overrides": []

  9940.           },

  9941.           "gridPos": {

  9942.             "h": 7,

  9943.             "w": 12,

  9944.             "x": 12,

  9945.             "y": 162

  9946.           },

  9947.           "id": 41,

  9948.           "links": [],

  9949.           "options": {

  9950.             "legend": {

  9951.               "calcs": [],

  9952.               "displayMode": "list",

  9953.               "placement": "bottom",

  9954.               "showLegend": true

  9955.             },

  9956.             "tooltip": {

  9957.               "mode": "single",

  9958.               "sort": "none"

  9959.             }

  9960.           },

  9961.           "pluginVersion": "8.4.3",

  9962.           "targets": [

  9963.             {

  9964.               "datasource": {

  9965.                 "type": "prometheus",

  9966.                 "uid": "${DS_PROMETHEUS}"

  9967.               },

  9968.               "exemplar": true,

  9969.               "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  9970.               "format": "time_series",

  9971.               "interval": "",

  9972.               "intervalFactor": 2,

  9973.               "legendFormat": "{{stream_name}}",

  9974.               "refId": "A",

  9975.               "step": 20

  9976.             }

  9977.           ],

  9978.           "title": "Rate of outgoing RDATA commands, by stream",

  9979.           "type": "timeseries"

  9980.         },

  9981.         {

  9982.           "datasource": {

  9983.             "type": "prometheus",

  9984.             "uid": "${DS_PROMETHEUS}"

  9985.           },

  9986.           "fieldConfig": {

  9987.             "defaults": {

  9988.               "color": {

  9989.                 "mode": "palette-classic"

  9990.               },

  9991.               "custom": {

  9992.                 "axisCenteredZero": false,

  9993.                 "axisColorMode": "text",

  9994.                 "axisLabel": "",

  9995.                 "axisPlacement": "auto",

  9996.                 "barAlignment": 0,

  9997.                 "drawStyle": "line",

  9998.                 "fillOpacity": 10,

  9999.                 "gradientMode": "none",

  10000.                 "hideFrom": {

  10001.                   "legend": false,

  10002.                   "tooltip": false,

  10003.                   "viz": false

  10004.                 },

  10005.                 "lineInterpolation": "linear",

  10006.                 "lineWidth": 1,

  10007.                 "pointSize": 5,

  10008.                 "scaleDistribution": {

  10009.                   "type": "linear"

  10010.                 },

  10011.                 "showPoints": "never",

  10012.                 "spanNulls": true,

  10013.                 "stacking": {

  10014.                   "group": "A",

  10015.                   "mode": "none"

  10016.                 },

  10017.                 "thresholdsStyle": {

  10018.                   "mode": "off"

  10019.                 }

  10020.               },

  10021.               "links": [],

  10022.               "mappings": [],

  10023.               "min": 0,

  10024.               "thresholds": {

  10025.                 "mode": "absolute",

  10026.                 "steps": [

  10027.                   {

  10028.                     "color": "green"

  10029.                   },

  10030.                   {

  10031.                     "color": "red",

  10032.                     "value": 80

  10033.                   }

  10034.                 ]

  10035.               },

  10036.               "unit": "hertz"

  10037.             },

  10038.             "overrides": []

  10039.           },

  10040.           "gridPos": {

  10041.             "h": 7,

  10042.             "w": 12,

  10043.             "x": 0,

  10044.             "y": 169

  10045.           },

  10046.           "id": 42,

  10047.           "links": [],

  10048.           "options": {

  10049.             "legend": {

  10050.               "calcs": [],

  10051.               "displayMode": "list",

  10052.               "placement": "bottom",

  10053.               "showLegend": true

  10054.             },

  10055.             "tooltip": {

  10056.               "mode": "single",

  10057.               "sort": "none"

  10058.             }

  10059.           },

  10060.           "pluginVersion": "8.4.3",

  10061.           "targets": [

  10062.             {

  10063.               "datasource": {

  10064.                 "type": "prometheus",

  10065.                 "uid": "${DS_PROMETHEUS}"

  10066.               },

  10067.               "exemplar": true,

  10068.               "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",

  10069.               "format": "time_series",

  10070.               "interval": "",

  10071.               "intervalFactor": 2,

  10072.               "legendFormat": "{{job}}-{{index}} {{command}}",

  10073.               "refId": "A",

  10074.               "step": 20

  10075.             }

  10076.           ],

  10077.           "title": "Rate of incoming commands (including echoes)",

  10078.           "type": "timeseries"

  10079.         },

  10080.         {

  10081.           "datasource": {

  10082.             "type": "prometheus",

  10083.             "uid": "${DS_PROMETHEUS}"

  10084.           },

  10085.           "fieldConfig": {

  10086.             "defaults": {

  10087.               "color": {

  10088.                 "mode": "palette-classic"

  10089.               },

  10090.               "custom": {

  10091.                 "axisCenteredZero": false,

  10092.                 "axisColorMode": "text",

  10093.                 "axisLabel": "",

  10094.                 "axisPlacement": "auto",

  10095.                 "axisSoftMin": 1,

  10096.                 "barAlignment": 0,

  10097.                 "drawStyle": "line",

  10098.                 "fillOpacity": 10,

  10099.                 "gradientMode": "none",

  10100.                 "hideFrom": {

  10101.                   "legend": false,

  10102.                   "tooltip": false,

  10103.                   "viz": false

  10104.                 },

  10105.                 "lineInterpolation": "linear",

  10106.                 "lineWidth": 1,

  10107.                 "pointSize": 5,

  10108.                 "scaleDistribution": {

  10109.                   "type": "linear"

  10110.                 },

  10111.                 "showPoints": "never",

  10112.                 "spanNulls": true,

  10113.                 "stacking": {

  10114.                   "group": "A",

  10115.                   "mode": "none"

  10116.                 },

  10117.                 "thresholdsStyle": {

  10118.                   "mode": "off"

  10119.                 }

  10120.               },

  10121.               "links": [],

  10122.               "mappings": [],

  10123.               "min": 0,

  10124.               "thresholds": {

  10125.                 "mode": "absolute",

  10126.                 "steps": [

  10127.                   {

  10128.                     "color": "green"

  10129.                   },

  10130.                   {

  10131.                     "color": "red",

  10132.                     "value": 80

  10133.                   }

  10134.                 ]

  10135.               },

  10136.               "unit": "hertz"

  10137.             },

  10138.             "overrides": []

  10139.           },

  10140.           "gridPos": {

  10141.             "h": 7,

  10142.             "w": 12,

  10143.             "x": 12,

  10144.             "y": 169

  10145.           },

  10146.           "id": 220,

  10147.           "links": [],

  10148.           "options": {

  10149.             "legend": {

  10150.               "calcs": [],

  10151.               "displayMode": "list",

  10152.               "placement": "bottom",

  10153.               "showLegend": true

  10154.             },

  10155.             "tooltip": {

  10156.               "mode": "single",

  10157.               "sort": "none"

  10158.             }

  10159.           },

  10160.           "pluginVersion": "8.4.3",

  10161.           "targets": [

  10162.             {

  10163.               "datasource": {

  10164.                 "type": "prometheus",

  10165.                 "uid": "${DS_PROMETHEUS}"

  10166.               },

  10167.               "exemplar": true,

  10168.               "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  10169.               "format": "time_series",

  10170.               "interval": "",

  10171.               "intervalFactor": 1,

  10172.               "legendFormat": "{{job}}-{{index}} {{stream_name}}",

  10173.               "refId": "A",

  10174.               "step": 20

  10175.             }

  10176.           ],

  10177.           "title": "Rate of incoming RDATA commands (excluding echoes), by stream",

  10178.           "type": "timeseries"

  10179.         },

  10180.         {

  10181.           "aliasColors": {},

  10182.           "bars": false,

  10183.           "dashLength": 10,

  10184.           "dashes": false,

  10185.           "datasource": {

  10186.             "type": "prometheus",

  10187.             "uid": "${DS_PROMETHEUS}"

  10188.           },

  10189.           "description": "",

  10190.           "fieldConfig": {

  10191.             "defaults": {

  10192.               "links": []

  10193.             },

  10194.             "overrides": []

  10195.           },

  10196.           "fill": 1,

  10197.           "fillGradient": 0,

  10198.           "gridPos": {

  10199.             "h": 7,

  10200.             "w": 12,

  10201.             "x": 0,

  10202.             "y": 176

  10203.           },

  10204.           "hiddenSeries": false,

  10205.           "id": 144,

  10206.           "legend": {

  10207.             "avg": false,

  10208.             "current": false,

  10209.             "max": false,

  10210.             "min": false,

  10211.             "show": true,

  10212.             "total": false,

  10213.             "values": false

  10214.           },

  10215.           "lines": true,

  10216.           "linewidth": 1,

  10217.           "nullPointMode": "null",

  10218.           "options": {

  10219.             "alertThreshold": true

  10220.           },

  10221.           "percentage": false,

  10222.           "pluginVersion": "9.2.2",

  10223.           "pointradius": 2,

  10224.           "points": false,

  10225.           "renderer": "flot",

  10226.           "seriesOverrides": [],

  10227.           "spaceLength": 10,

  10228.           "stack": false,

  10229.           "steppedLine": false,

  10230.           "targets": [

  10231.             {

  10232.               "datasource": {

  10233.                 "type": "prometheus",

  10234.                 "uid": "${DS_PROMETHEUS}"

  10235.               },

  10236.               "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  10237.               "interval": "",

  10238.               "legendFormat": "{{stream_name}} {{job}}-{{index}}",

  10239.               "refId": "A"

  10240.             }

  10241.           ],

  10242.           "thresholds": [],

  10243.           "timeRegions": [],

  10244.           "title": "Queued incoming RDATA commands, by stream",

  10245.           "tooltip": {

  10246.             "shared": false,

  10247.             "sort": 0,

  10248.             "value_type": "individual"

  10249.           },

  10250.           "type": "graph",

  10251.           "xaxis": {

  10252.             "mode": "time",

  10253.             "show": true,

  10254.             "values": []

  10255.           },

  10256.           "yaxes": [

  10257.             {

  10258.               "$$hashKey": "object:218",

  10259.               "format": "short",

  10260.               "logBase": 1,

  10261.               "show": true

  10262.             },

  10263.             {

  10264.               "$$hashKey": "object:219",

  10265.               "format": "short",

  10266.               "logBase": 1,

  10267.               "show": true

  10268.             }

  10269.           ],

  10270.           "yaxis": {

  10271.             "align": false

  10272.           }

  10273.         },

  10274.         {

  10275.           "aliasColors": {},

  10276.           "bars": false,

  10277.           "dashLength": 10,

  10278.           "dashes": false,

  10279.           "datasource": {

  10280.             "type": "prometheus",

  10281.             "uid": "${DS_PROMETHEUS}"

  10282.           },

  10283.           "fieldConfig": {

  10284.             "defaults": {

  10285.               "links": []

  10286.             },

  10287.             "overrides": []

  10288.           },

  10289.           "fill": 1,

  10290.           "fillGradient": 0,

  10291.           "gridPos": {

  10292.             "h": 7,

  10293.             "w": 12,

  10294.             "x": 12,

  10295.             "y": 176

  10296.           },

  10297.           "hiddenSeries": false,

  10298.           "id": 115,

  10299.           "legend": {

  10300.             "avg": false,

  10301.             "current": false,

  10302.             "max": false,

  10303.             "min": false,

  10304.             "show": true,

  10305.             "total": false,

  10306.             "values": false

  10307.           },

  10308.           "lines": true,

  10309.           "linewidth": 1,

  10310.           "links": [],

  10311.           "nullPointMode": "null",

  10312.           "options": {

  10313.             "alertThreshold": true

  10314.           },

  10315.           "paceLength": 10,

  10316.           "percentage": false,

  10317.           "pluginVersion": "9.2.2",

  10318.           "pointradius": 5,

  10319.           "points": false,

  10320.           "renderer": "flot",

  10321.           "seriesOverrides": [],

  10322.           "spaceLength": 10,

  10323.           "stack": false,

  10324.           "steppedLine": false,

  10325.           "targets": [

  10326.             {

  10327.               "datasource": {

  10328.                 "uid": "${DS_PROMETHEUS}"

  10329.               },

  10330.               "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  10331.               "format": "time_series",

  10332.               "intervalFactor": 1,

  10333.               "legendFormat": "{{job}}-{{index}} {{reason_type}}",

  10334.               "refId": "A"

  10335.             }

  10336.           ],

  10337.           "thresholds": [],

  10338.           "timeRegions": [],

  10339.           "title": "Replication connection close reasons",

  10340.           "tooltip": {

  10341.             "shared": true,

  10342.             "sort": 0,

  10343.             "value_type": "individual"

  10344.           },

  10345.           "type": "graph",

  10346.           "xaxis": {

  10347.             "mode": "time",

  10348.             "show": true,

  10349.             "values": []

  10350.           },

  10351.           "yaxes": [

  10352.             {

  10353.               "$$hashKey": "object:260",

  10354.               "format": "hertz",

  10355.               "logBase": 1,

  10356.               "show": true

  10357.             },

  10358.             {

  10359.               "$$hashKey": "object:261",

  10360.               "format": "short",

  10361.               "logBase": 1,

  10362.               "show": true

  10363.             }

  10364.           ],

  10365.           "yaxis": {

  10366.             "align": false

  10367.           }

  10368.         },

  10369.         {

  10370.           "aliasColors": {},

  10371.           "bars": false,

  10372.           "dashLength": 10,

  10373.           "dashes": false,

  10374.           "datasource": {

  10375.             "uid": "${DS_PROMETHEUS}"

  10376.           },

  10377.           "fieldConfig": {

  10378.             "defaults": {

  10379.               "links": []

  10380.             },

  10381.             "overrides": []

  10382.           },

  10383.           "fill": 1,

  10384.           "fillGradient": 0,

  10385.           "gridPos": {

  10386.             "h": 7,

  10387.             "w": 12,

  10388.             "x": 0,

  10389.             "y": 183

  10390.           },

  10391.           "hiddenSeries": false,

  10392.           "id": 113,

  10393.           "legend": {

  10394.             "avg": false,

  10395.             "current": false,

  10396.             "max": false,

  10397.             "min": false,

  10398.             "show": true,

  10399.             "total": false,

  10400.             "values": false

  10401.           },

  10402.           "lines": true,

  10403.           "linewidth": 1,

  10404.           "links": [],

  10405.           "nullPointMode": "null",

  10406.           "options": {

  10407.             "alertThreshold": true

  10408.           },

  10409.           "paceLength": 10,

  10410.           "percentage": false,

  10411.           "pluginVersion": "9.2.2",

  10412.           "pointradius": 5,

  10413.           "points": false,

  10414.           "renderer": "flot",

  10415.           "seriesOverrides": [],

  10416.           "spaceLength": 10,

  10417.           "stack": false,

  10418.           "steppedLine": false,

  10419.           "targets": [

  10420.             {

  10421.               "datasource": {

  10422.                 "uid": "${DS_PROMETHEUS}"

  10423.               },

  10424.               "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",

  10425.               "format": "time_series",

  10426.               "intervalFactor": 1,

  10427.               "legendFormat": "{{job}}-{{index}} {{stream_name}}",

  10428.               "refId": "A"

  10429.             },

  10430.             {

  10431.               "datasource": {

  10432.                 "uid": "${DS_PROMETHEUS}"

  10433.               },

  10434.               "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",

  10435.               "format": "time_series",

  10436.               "intervalFactor": 1,

  10437.               "legendFormat": "{{job}}-{{index}}",

  10438.               "refId": "B"

  10439.             }

  10440.           ],

  10441.           "thresholds": [],

  10442.           "timeRegions": [],

  10443.           "title": "Replication connections",

  10444.           "tooltip": {

  10445.             "shared": true,

  10446.             "sort": 0,

  10447.             "value_type": "individual"

  10448.           },

  10449.           "type": "graph",

  10450.           "xaxis": {

  10451.             "mode": "time",

  10452.             "show": true,

  10453.             "values": []

  10454.           },

  10455.           "yaxes": [

  10456.             {

  10457.               "format": "short",

  10458.               "logBase": 1,

  10459.               "min": "0",

  10460.               "show": true

  10461.             },

  10462.             {

  10463.               "format": "short",

  10464.               "logBase": 1,

  10465.               "show": true

  10466.             }

  10467.           ],

  10468.           "yaxis": {

  10469.             "align": false

  10470.           }

  10471.         }

  10472.       ],

  10473.       "targets": [

  10474.         {

  10475.           "datasource": {

  10476.             "type": "prometheus",

  10477.             "uid": "${DS_PROMETHEUS}"

  10478.           },

  10479.           "refId": "A"

  10480.         }

  10481.       ],

  10482.       "title": "Replication",

  10483.       "type": "row"

  10484.     },

  10485.     {

  10486.       "collapsed": true,

  10487.       "datasource": {

  10488.         "type": "prometheus",

  10489.         "uid": "${DS_PROMETHEUS}"

  10490.       },

  10491.       "gridPos": {

  10492.         "h": 1,

  10493.         "w": 24,

  10494.         "x": 0,

  10495.         "y": 40

  10496.       },

  10497.       "id": 69,

  10498.       "panels": [

  10499.         {

  10500.           "aliasColors": {},

  10501.           "bars": false,

  10502.           "dashLength": 10,

  10503.           "dashes": false,

  10504.           "datasource": {

  10505.             "uid": "${DS_PROMETHEUS}"

  10506.           },

  10507.           "fieldConfig": {

  10508.             "defaults": {

  10509.               "links": []

  10510.             },

  10511.             "overrides": []

  10512.           },

  10513.           "fill": 1,

  10514.           "fillGradient": 0,

  10515.           "gridPos": {

  10516.             "h": 9,

  10517.             "w": 12,

  10518.             "x": 0,

  10519.             "y": 163

  10520.           },

  10521.           "hiddenSeries": false,

  10522.           "id": 67,

  10523.           "legend": {

  10524.             "avg": false,

  10525.             "current": false,

  10526.             "max": false,

  10527.             "min": false,

  10528.             "show": true,

  10529.             "total": false,

  10530.             "values": false

  10531.           },

  10532.           "lines": true,

  10533.           "linewidth": 1,

  10534.           "links": [],

  10535.           "nullPointMode": "connected",

  10536.           "options": {

  10537.             "alertThreshold": true

  10538.           },

  10539.           "paceLength": 10,

  10540.           "percentage": false,

  10541.           "pluginVersion": "9.2.2",

  10542.           "pointradius": 5,

  10543.           "points": false,

  10544.           "renderer": "flot",

  10545.           "seriesOverrides": [],

  10546.           "spaceLength": 10,

  10547.           "stack": false,

  10548.           "steppedLine": false,

  10549.           "targets": [

  10550.             {

  10551.               "datasource": {

  10552.                 "uid": "${DS_PROMETHEUS}"

  10553.               },

  10554.               "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  10555.               "format": "time_series",

  10556.               "interval": "",

  10557.               "intervalFactor": 1,

  10558.               "legendFormat": "{{job}}-{{index}} {{name}}",

  10559.               "refId": "A"

  10560.             }

  10561.           ],

  10562.           "thresholds": [],

  10563.           "timeRegions": [],

  10564.           "title": "Event processing lag",

  10565.           "tooltip": {

  10566.             "shared": true,

  10567.             "sort": 0,

  10568.             "value_type": "individual"

  10569.           },

  10570.           "type": "graph",

  10571.           "xaxis": {

  10572.             "mode": "time",

  10573.             "show": true,

  10574.             "values": []

  10575.           },

  10576.           "yaxes": [

  10577.             {

  10578.               "format": "short",

  10579.               "label": "events",

  10580.               "logBase": 1,

  10581.               "min": "0",

  10582.               "show": true

  10583.             },

  10584.             {

  10585.               "format": "short",

  10586.               "logBase": 1,

  10587.               "show": true

  10588.             }

  10589.           ],

  10590.           "yaxis": {

  10591.             "align": false

  10592.           }

  10593.         },

  10594.         {

  10595.           "aliasColors": {},

  10596.           "bars": false,

  10597.           "dashLength": 10,

  10598.           "dashes": false,

  10599.           "datasource": {

  10600.             "uid": "${DS_PROMETHEUS}"

  10601.           },

  10602.           "fieldConfig": {

  10603.             "defaults": {

  10604.               "links": []

  10605.             },

  10606.             "overrides": []

  10607.           },

  10608.           "fill": 1,

  10609.           "fillGradient": 0,

  10610.           "gridPos": {

  10611.             "h": 9,

  10612.             "w": 12,

  10613.             "x": 12,

  10614.             "y": 163

  10615.           },

  10616.           "hiddenSeries": false,

  10617.           "id": 71,

  10618.           "legend": {

  10619.             "avg": false,

  10620.             "current": false,

  10621.             "max": false,

  10622.             "min": false,

  10623.             "show": true,

  10624.             "total": false,

  10625.             "values": false

  10626.           },

  10627.           "lines": true,

  10628.           "linewidth": 1,

  10629.           "links": [],

  10630.           "nullPointMode": "connected",

  10631.           "options": {

  10632.             "alertThreshold": true

  10633.           },

  10634.           "paceLength": 10,

  10635.           "percentage": false,

  10636.           "pluginVersion": "9.2.2",

  10637.           "pointradius": 5,

  10638.           "points": false,

  10639.           "renderer": "flot",

  10640.           "seriesOverrides": [],

  10641.           "spaceLength": 10,

  10642.           "stack": false,

  10643.           "steppedLine": false,

  10644.           "targets": [

  10645.             {

  10646.               "datasource": {

  10647.                 "uid": "${DS_PROMETHEUS}"

  10648.               },

  10649.               "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",

  10650.               "format": "time_series",

  10651.               "hide": false,

  10652.               "interval": "",

  10653.               "intervalFactor": 1,

  10654.               "legendFormat": "{{job}}-{{index}} {{name}}",

  10655.               "refId": "B"

  10656.             }

  10657.           ],

  10658.           "thresholds": [],

  10659.           "timeRegions": [],

  10660.           "title": "Age of last processed event",

  10661.           "tooltip": {

  10662.             "shared": true,

  10663.             "sort": 0,

  10664.             "value_type": "individual"

  10665.           },

  10666.           "type": "graph",

  10667.           "xaxis": {

  10668.             "mode": "time",

  10669.             "show": true,

  10670.             "values": []

  10671.           },

  10672.           "yaxes": [

  10673.             {

  10674.               "format": "ms",

  10675.               "logBase": 1,

  10676.               "min": "0",

  10677.               "show": true

  10678.             },

  10679.             {

  10680.               "format": "short",

  10681.               "logBase": 1,

  10682.               "show": true

  10683.             }

  10684.           ],

  10685.           "yaxis": {

  10686.             "align": false

  10687.           }

  10688.         },

  10689.         {

  10690.           "aliasColors": {},

  10691.           "bars": false,

  10692.           "dashLength": 10,

  10693.           "dashes": false,

  10694.           "datasource": {

  10695.             "uid": "${DS_PROMETHEUS}"

  10696.           },

  10697.           "fieldConfig": {

  10698.             "defaults": {

  10699.               "links": []

  10700.             },

  10701.             "overrides": []

  10702.           },

  10703.           "fill": 1,

  10704.           "fillGradient": 0,

  10705.           "gridPos": {

  10706.             "h": 9,

  10707.             "w": 12,

  10708.             "x": 0,

  10709.             "y": 172

  10710.           },

  10711.           "hiddenSeries": false,

  10712.           "id": 121,

  10713.           "interval": "",

  10714.           "legend": {

  10715.             "avg": false,

  10716.             "current": false,

  10717.             "max": false,

  10718.             "min": false,

  10719.             "show": true,

  10720.             "total": false,

  10721.             "values": false

  10722.           },

  10723.           "lines": true,

  10724.           "linewidth": 1,

  10725.           "links": [],

  10726.           "nullPointMode": "connected",

  10727.           "options": {

  10728.             "alertThreshold": true

  10729.           },

  10730.           "paceLength": 10,

  10731.           "percentage": false,

  10732.           "pluginVersion": "9.2.2",

  10733.           "pointradius": 5,

  10734.           "points": false,

  10735.           "renderer": "flot",

  10736.           "seriesOverrides": [],

  10737.           "spaceLength": 10,

  10738.           "stack": false,

  10739.           "steppedLine": false,

  10740.           "targets": [

  10741.             {

  10742.               "datasource": {

  10743.                 "uid": "${DS_PROMETHEUS}"

  10744.               },

  10745.               "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",

  10746.               "format": "time_series",

  10747.               "hide": false,

  10748.               "interval": "",

  10749.               "intervalFactor": 1,

  10750.               "legendFormat": "{{job}}-{{index}} {{name}}",

  10751.               "refId": "B"

  10752.             }

  10753.           ],

  10754.           "thresholds": [],

  10755.           "timeRegions": [],

  10756.           "title": "Event processing catchup rate",

  10757.           "tooltip": {

  10758.             "shared": true,

  10759.             "sort": 0,

  10760.             "value_type": "individual"

  10761.           },

  10762.           "type": "graph",

  10763.           "xaxis": {

  10764.             "mode": "time",

  10765.             "show": true,

  10766.             "values": []

  10767.           },

  10768.           "yaxes": [

  10769.             {

  10770.               "format": "none",

  10771.               "label": "fallbehind(-) / catchup(+):  s/sec",

  10772.               "logBase": 1,

  10773.               "show": true

  10774.             },

  10775.             {

  10776.               "format": "short",

  10777.               "logBase": 1,

  10778.               "show": true

  10779.             }

  10780.           ],

  10781.           "yaxis": {

  10782.             "align": false

  10783.           }

  10784.         }

  10785.       ],

  10786.       "targets": [

  10787.         {

  10788.           "datasource": {

  10789.             "type": "prometheus",

  10790.             "uid": "${DS_PROMETHEUS}"

  10791.           },

  10792.           "refId": "A"

  10793.         }

  10794.       ],

  10795.       "title": "Event processing loop positions",

  10796.       "type": "row"

  10797.     },

  10798.     {

  10799.       "collapsed": true,

  10800.       "datasource": {

  10801.         "type": "prometheus",

  10802.         "uid": "${DS_PROMETHEUS}"

  10803.       },

  10804.       "gridPos": {

  10805.         "h": 1,

  10806.         "w": 24,

  10807.         "x": 0,

  10808.         "y": 41

  10809.       },

  10810.       "id": 126,

  10811.       "panels": [

  10812.         {

  10813.           "cards": {

  10814.             "cardPadding": 0

  10815.           },

  10816.           "color": {

  10817.             "cardColor": "#B877D9",

  10818.             "colorScale": "sqrt",

  10819.             "colorScheme": "interpolateInferno",

  10820.             "exponent": 0.5,

  10821.             "min": 0,

  10822.             "mode": "opacity"

  10823.           },

  10824.           "dataFormat": "tsbuckets",

  10825.           "datasource": {

  10826.             "uid": "${DS_PROMETHEUS}"

  10827.           },

  10828.           "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",

  10829.           "fieldConfig": {

  10830.             "defaults": {

  10831.               "custom": {

  10832.                 "hideFrom": {

  10833.                   "legend": false,

  10834.                   "tooltip": false,

  10835.                   "viz": false

  10836.                 },

  10837.                 "scaleDistribution": {

  10838.                   "type": "linear"

  10839.                 }

  10840.               }

  10841.             },

  10842.             "overrides": []

  10843.           },

  10844.           "gridPos": {

  10845.             "h": 8,

  10846.             "w": 12,

  10847.             "x": 0,

  10848.             "y": 42

  10849.           },

  10850.           "heatmap": {},

  10851.           "hideZeroBuckets": true,

  10852.           "highlightCards": true,

  10853.           "id": 122,

  10854.           "legend": {

  10855.             "show": true

  10856.           },

  10857.           "links": [],

  10858.           "options": {

  10859.             "calculate": false,

  10860.             "calculation": {},

  10861.             "cellGap": 1,

  10862.             "cellValues": {},

  10863.             "color": {

  10864.               "exponent": 0.5,

  10865.               "fill": "#B877D9",

  10866.               "min": 0,

  10867.               "mode": "opacity",

  10868.               "reverse": false,

  10869.               "scale": "exponential",

  10870.               "scheme": "Oranges",

  10871.               "steps": 128

  10872.             },

  10873.             "exemplars": {

  10874.               "color": "rgba(255,0,255,0.7)"

  10875.             },

  10876.             "filterValues": {

  10877.               "le": 1e-9

  10878.             },

  10879.             "legend": {

  10880.               "show": true

  10881.             },

  10882.             "rowsFrame": {

  10883.               "layout": "auto"

  10884.             },

  10885.             "showValue": "never",

  10886.             "tooltip": {

  10887.               "show": true,

  10888.               "yHistogram": true

  10889.             },

  10890.             "yAxis": {

  10891.               "axisPlacement": "left",

  10892.               "decimals": 0,

  10893.               "reverse": false,

  10894.               "unit": "short"

  10895.             }

  10896.           },

  10897.           "pluginVersion": "9.2.2",

  10898.           "reverseYBuckets": false,

  10899.           "targets": [

  10900.             {

  10901.               "datasource": {

  10902.                 "uid": "${DS_PROMETHEUS}"

  10903.               },

  10904.               "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",

  10905.               "format": "heatmap",

  10906.               "intervalFactor": 1,

  10907.               "legendFormat": "{{le}}",

  10908.               "refId": "A"

  10909.             }

  10910.           ],

  10911.           "title": "Number of rooms, by number of forward extremities in room",

  10912.           "tooltip": {

  10913.             "show": true,

  10914.             "showHistogram": true

  10915.           },

  10916.           "type": "heatmap",

  10917.           "xAxis": {

  10918.             "show": true

  10919.           },

  10920.           "yAxis": {

  10921.             "decimals": 0,

  10922.             "format": "short",

  10923.             "logBase": 1,

  10924.             "show": true

  10925.           },

  10926.           "yBucketBound": "auto"

  10927.         },

  10928.         {

  10929.           "aliasColors": {},

  10930.           "bars": false,

  10931.           "dashLength": 10,

  10932.           "dashes": false,

  10933.           "datasource": {

  10934.             "uid": "${DS_PROMETHEUS}"

  10935.           },

  10936.           "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",

  10937.           "fieldConfig": {

  10938.             "defaults": {

  10939.               "links": []

  10940.             },

  10941.             "overrides": []

  10942.           },

  10943.           "fill": 0,

  10944.           "fillGradient": 0,

  10945.           "gridPos": {

  10946.             "h": 8,

  10947.             "w": 12,

  10948.             "x": 12,

  10949.             "y": 42

  10950.           },

  10951.           "hiddenSeries": false,

  10952.           "id": 124,

  10953.           "interval": "",

  10954.           "legend": {

  10955.             "avg": false,

  10956.             "current": false,

  10957.             "max": false,

  10958.             "min": false,

  10959.             "show": true,

  10960.             "total": false,

  10961.             "values": false

  10962.           },

  10963.           "lines": true,

  10964.           "linewidth": 1,

  10965.           "links": [],

  10966.           "nullPointMode": "connected",

  10967.           "options": {

  10968.             "alertThreshold": true

  10969.           },

  10970.           "percentage": false,

  10971.           "pluginVersion": "9.2.2",

  10972.           "pointradius": 2,

  10973.           "points": false,

  10974.           "renderer": "flot",

  10975.           "seriesOverrides": [],

  10976.           "spaceLength": 10,

  10977.           "stack": false,

  10978.           "steppedLine": false,

  10979.           "targets": [

  10980.             {

  10981.               "datasource": {

  10982.                 "uid": "${DS_PROMETHEUS}"

  10983.               },

  10984.               "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",

  10985.               "format": "heatmap",

  10986.               "interval": "",

  10987.               "intervalFactor": 1,

  10988.               "legendFormat": "{{le}}",

  10989.               "refId": "A"

  10990.             }

  10991.           ],

  10992.           "thresholds": [],

  10993.           "timeRegions": [],

  10994.           "title": "Room counts, by number of extremities",

  10995.           "tooltip": {

  10996.             "shared": true,

  10997.             "sort": 2,

  10998.             "value_type": "individual"

  10999.           },

  11000.           "type": "graph",

  11001.           "xaxis": {

  11002.             "mode": "time",

  11003.             "show": true,

  11004.             "values": []

  11005.           },

  11006.           "yaxes": [

  11007.             {

  11008.               "format": "none",

  11009.               "label": "Number of rooms",

  11010.               "logBase": 10,

  11011.               "show": true

  11012.             },

  11013.             {

  11014.               "format": "short",

  11015.               "logBase": 1,

  11016.               "show": false

  11017.             }

  11018.           ],

  11019.           "yaxis": {

  11020.             "align": false

  11021.           }

  11022.         },

  11023.         {

  11024.           "cards": {

  11025.             "cardPadding": 0

  11026.           },

  11027.           "color": {

  11028.             "cardColor": "#5794F2",

  11029.             "colorScale": "sqrt",

  11030.             "colorScheme": "interpolateInferno",

  11031.             "exponent": 0.5,

  11032.             "min": 0,

  11033.             "mode": "opacity"

  11034.           },

  11035.           "dataFormat": "tsbuckets",

  11036.           "datasource": {

  11037.             "uid": "${DS_PROMETHEUS}"

  11038.           },

  11039.           "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",

  11040.           "fieldConfig": {

  11041.             "defaults": {

  11042.               "custom": {

  11043.                 "hideFrom": {

  11044.                   "legend": false,

  11045.                   "tooltip": false,

  11046.                   "viz": false

  11047.                 },

  11048.                 "scaleDistribution": {

  11049.                   "type": "linear"

  11050.                 }

  11051.               }

  11052.             },

  11053.             "overrides": []

  11054.           },

  11055.           "gridPos": {

  11056.             "h": 8,

  11057.             "w": 12,

  11058.             "x": 0,

  11059.             "y": 50

  11060.           },

  11061.           "heatmap": {},

  11062.           "hideZeroBuckets": true,

  11063.           "highlightCards": true,

  11064.           "id": 127,

  11065.           "legend": {

  11066.             "show": true

  11067.           },

  11068.           "links": [],

  11069.           "options": {

  11070.             "calculate": false,

  11071.             "calculation": {},

  11072.             "cellGap": 1,

  11073.             "cellValues": {},

  11074.             "color": {

  11075.               "exponent": 0.5,

  11076.               "fill": "#5794F2",

  11077.               "min": 0,

  11078.               "mode": "opacity",

  11079.               "reverse": false,

  11080.               "scale": "exponential",

  11081.               "scheme": "Oranges",

  11082.               "steps": 128

  11083.             },

  11084.             "exemplars": {

  11085.               "color": "rgba(255,0,255,0.7)"

  11086.             },

  11087.             "filterValues": {

  11088.               "le": 1e-9

  11089.             },

  11090.             "legend": {

  11091.               "show": true

  11092.             },

  11093.             "rowsFrame": {

  11094.               "layout": "auto"

  11095.             },

  11096.             "showValue": "never",

  11097.             "tooltip": {

  11098.               "show": true,

  11099.               "yHistogram": true

  11100.             },

  11101.             "yAxis": {

  11102.               "axisPlacement": "left",

  11103.               "decimals": 0,

  11104.               "reverse": false,

  11105.               "unit": "short"

  11106.             }

  11107.           },

  11108.           "pluginVersion": "9.2.2",

  11109.           "reverseYBuckets": false,

  11110.           "targets": [

  11111.             {

  11112.               "datasource": {

  11113.                 "uid": "${DS_PROMETHEUS}"

  11114.               },

  11115.               "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",

  11116.               "format": "heatmap",

  11117.               "intervalFactor": 1,

  11118.               "legendFormat": "{{le}}",

  11119.               "refId": "A"

  11120.             }

  11121.           ],

  11122.           "title": "Events persisted, by number of forward extremities in room (heatmap)",

  11123.           "tooltip": {

  11124.             "show": true,

  11125.             "showHistogram": true

  11126.           },

  11127.           "type": "heatmap",

  11128.           "xAxis": {

  11129.             "show": true

  11130.           },

  11131.           "yAxis": {

  11132.             "decimals": 0,

  11133.             "format": "short",

  11134.             "logBase": 1,

  11135.             "show": true

  11136.           },

  11137.           "yBucketBound": "auto"

  11138.         },

  11139.         {

  11140.           "aliasColors": {},

  11141.           "bars": false,

  11142.           "dashLength": 10,

  11143.           "dashes": false,

  11144.           "datasource": {

  11145.             "uid": "${DS_PROMETHEUS}"

  11146.           },

  11147.           "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",

  11148.           "fieldConfig": {

  11149.             "defaults": {

  11150.               "links": []

  11151.             },

  11152.             "overrides": []

  11153.           },

  11154.           "fill": 1,

  11155.           "fillGradient": 0,

  11156.           "gridPos": {

  11157.             "h": 8,

  11158.             "w": 12,

  11159.             "x": 12,

  11160.             "y": 50

  11161.           },

  11162.           "hiddenSeries": false,

  11163.           "id": 128,

  11164.           "legend": {

  11165.             "avg": false,

  11166.             "current": false,

  11167.             "max": false,

  11168.             "min": false,

  11169.             "show": true,

  11170.             "total": false,

  11171.             "values": false

  11172.           },

  11173.           "lines": true,

  11174.           "linewidth": 1,

  11175.           "links": [],

  11176.           "nullPointMode": "null",

  11177.           "options": {

  11178.             "alertThreshold": true

  11179.           },

  11180.           "percentage": false,

  11181.           "pluginVersion": "9.2.2",

  11182.           "pointradius": 2,

  11183.           "points": false,

  11184.           "renderer": "flot",

  11185.           "seriesOverrides": [],

  11186.           "spaceLength": 10,

  11187.           "stack": false,

  11188.           "steppedLine": false,

  11189.           "targets": [

  11190.             {

  11191.               "datasource": {

  11192.                 "uid": "${DS_PROMETHEUS}"

  11193.               },

  11194.               "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11195.               "format": "time_series",

  11196.               "intervalFactor": 1,

  11197.               "legendFormat": "50%",

  11198.               "refId": "A"

  11199.             },

  11200.             {

  11201.               "datasource": {

  11202.                 "uid": "${DS_PROMETHEUS}"

  11203.               },

  11204.               "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11205.               "format": "time_series",

  11206.               "intervalFactor": 1,

  11207.               "legendFormat": "75%",

  11208.               "refId": "B"

  11209.             },

  11210.             {

  11211.               "datasource": {

  11212.                 "uid": "${DS_PROMETHEUS}"

  11213.               },

  11214.               "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11215.               "format": "time_series",

  11216.               "intervalFactor": 1,

  11217.               "legendFormat": "90%",

  11218.               "refId": "C"

  11219.             },

  11220.             {

  11221.               "datasource": {

  11222.                 "uid": "${DS_PROMETHEUS}"

  11223.               },

  11224.               "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11225.               "format": "time_series",

  11226.               "intervalFactor": 1,

  11227.               "legendFormat": "99%",

  11228.               "refId": "D"

  11229.             }

  11230.           ],

  11231.           "thresholds": [],

  11232.           "timeRegions": [],

  11233.           "title": "Events persisted, by number of forward extremities in room (quantiles)",

  11234.           "tooltip": {

  11235.             "shared": true,

  11236.             "sort": 0,

  11237.             "value_type": "individual"

  11238.           },

  11239.           "type": "graph",

  11240.           "xaxis": {

  11241.             "mode": "time",

  11242.             "show": true,

  11243.             "values": []

  11244.           },

  11245.           "yaxes": [

  11246.             {

  11247.               "format": "short",

  11248.               "label": "Number of extremities in room",

  11249.               "logBase": 1,

  11250.               "min": "0",

  11251.               "show": true

  11252.             },

  11253.             {

  11254.               "format": "short",

  11255.               "logBase": 1,

  11256.               "show": true

  11257.             }

  11258.           ],

  11259.           "yaxis": {

  11260.             "align": false

  11261.           }

  11262.         },

  11263.         {

  11264.           "cards": {

  11265.             "cardPadding": 0

  11266.           },

  11267.           "color": {

  11268.             "cardColor": "#FF9830",

  11269.             "colorScale": "sqrt",

  11270.             "colorScheme": "interpolateInferno",

  11271.             "exponent": 0.5,

  11272.             "min": 0,

  11273.             "mode": "opacity"

  11274.           },

  11275.           "dataFormat": "tsbuckets",

  11276.           "datasource": {

  11277.             "uid": "${DS_PROMETHEUS}"

  11278.           },

  11279.           "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",

  11280.           "fieldConfig": {

  11281.             "defaults": {

  11282.               "custom": {

  11283.                 "hideFrom": {

  11284.                   "legend": false,

  11285.                   "tooltip": false,

  11286.                   "viz": false

  11287.                 },

  11288.                 "scaleDistribution": {

  11289.                   "type": "linear"

  11290.                 }

  11291.               }

  11292.             },

  11293.             "overrides": []

  11294.           },

  11295.           "gridPos": {

  11296.             "h": 8,

  11297.             "w": 12,

  11298.             "x": 0,

  11299.             "y": 58

  11300.           },

  11301.           "heatmap": {},

  11302.           "hideZeroBuckets": true,

  11303.           "highlightCards": true,

  11304.           "id": 129,

  11305.           "legend": {

  11306.             "show": true

  11307.           },

  11308.           "links": [],

  11309.           "options": {

  11310.             "calculate": false,

  11311.             "calculation": {},

  11312.             "cellGap": 1,

  11313.             "cellValues": {},

  11314.             "color": {

  11315.               "exponent": 0.5,

  11316.               "fill": "#FF9830",

  11317.               "min": 0,

  11318.               "mode": "opacity",

  11319.               "reverse": false,

  11320.               "scale": "exponential",

  11321.               "scheme": "Oranges",

  11322.               "steps": 128

  11323.             },

  11324.             "exemplars": {

  11325.               "color": "rgba(255,0,255,0.7)"

  11326.             },

  11327.             "filterValues": {

  11328.               "le": 1e-9

  11329.             },

  11330.             "legend": {

  11331.               "show": true

  11332.             },

  11333.             "rowsFrame": {

  11334.               "layout": "auto"

  11335.             },

  11336.             "showValue": "never",

  11337.             "tooltip": {

  11338.               "show": true,

  11339.               "yHistogram": true

  11340.             },

  11341.             "yAxis": {

  11342.               "axisPlacement": "left",

  11343.               "decimals": 0,

  11344.               "reverse": false,

  11345.               "unit": "short"

  11346.             }

  11347.           },

  11348.           "pluginVersion": "9.2.2",

  11349.           "reverseYBuckets": false,

  11350.           "targets": [

  11351.             {

  11352.               "datasource": {

  11353.                 "uid": "${DS_PROMETHEUS}"

  11354.               },

  11355.               "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",

  11356.               "format": "heatmap",

  11357.               "intervalFactor": 1,

  11358.               "legendFormat": "{{le}}",

  11359.               "refId": "A"

  11360.             }

  11361.           ],

  11362.           "title": "Events persisted, by number of stale forward extremities in room (heatmap)",

  11363.           "tooltip": {

  11364.             "show": true,

  11365.             "showHistogram": true

  11366.           },

  11367.           "type": "heatmap",

  11368.           "xAxis": {

  11369.             "show": true

  11370.           },

  11371.           "yAxis": {

  11372.             "decimals": 0,

  11373.             "format": "short",

  11374.             "logBase": 1,

  11375.             "show": true

  11376.           },

  11377.           "yBucketBound": "auto"

  11378.         },

  11379.         {

  11380.           "aliasColors": {},

  11381.           "bars": false,

  11382.           "dashLength": 10,

  11383.           "dashes": false,

  11384.           "datasource": {

  11385.             "uid": "${DS_PROMETHEUS}"

  11386.           },

  11387.           "description": "For  given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",

  11388.           "fieldConfig": {

  11389.             "defaults": {

  11390.               "links": []

  11391.             },

  11392.             "overrides": []

  11393.           },

  11394.           "fill": 1,

  11395.           "fillGradient": 0,

  11396.           "gridPos": {

  11397.             "h": 8,

  11398.             "w": 12,

  11399.             "x": 12,

  11400.             "y": 58

  11401.           },

  11402.           "hiddenSeries": false,

  11403.           "id": 130,

  11404.           "legend": {

  11405.             "avg": false,

  11406.             "current": false,

  11407.             "max": false,

  11408.             "min": false,

  11409.             "show": true,

  11410.             "total": false,

  11411.             "values": false

  11412.           },

  11413.           "lines": true,

  11414.           "linewidth": 1,

  11415.           "links": [],

  11416.           "nullPointMode": "null",

  11417.           "options": {

  11418.             "alertThreshold": true

  11419.           },

  11420.           "percentage": false,

  11421.           "pluginVersion": "9.2.2",

  11422.           "pointradius": 2,

  11423.           "points": false,

  11424.           "renderer": "flot",

  11425.           "seriesOverrides": [],

  11426.           "spaceLength": 10,

  11427.           "stack": false,

  11428.           "steppedLine": false,

  11429.           "targets": [

  11430.             {

  11431.               "datasource": {

  11432.                 "uid": "${DS_PROMETHEUS}"

  11433.               },

  11434.               "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11435.               "format": "time_series",

  11436.               "intervalFactor": 1,

  11437.               "legendFormat": "50%",

  11438.               "refId": "A"

  11439.             },

  11440.             {

  11441.               "datasource": {

  11442.                 "uid": "${DS_PROMETHEUS}"

  11443.               },

  11444.               "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11445.               "format": "time_series",

  11446.               "intervalFactor": 1,

  11447.               "legendFormat": "75%",

  11448.               "refId": "B"

  11449.             },

  11450.             {

  11451.               "datasource": {

  11452.                 "uid": "${DS_PROMETHEUS}"

  11453.               },

  11454.               "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11455.               "format": "time_series",

  11456.               "intervalFactor": 1,

  11457.               "legendFormat": "90%",

  11458.               "refId": "C"

  11459.             },

  11460.             {

  11461.               "datasource": {

  11462.                 "uid": "${DS_PROMETHEUS}"

  11463.               },

  11464.               "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",

  11465.               "format": "time_series",

  11466.               "intervalFactor": 1,

  11467.               "legendFormat": "99%",

  11468.               "refId": "D"

  11469.             }

  11470.           ],

  11471.           "thresholds": [],

  11472.           "timeRegions": [],

  11473.           "title": "Events persisted, by number of stale forward extremities in room (quantiles)",

  11474.           "tooltip": {

  11475.             "shared": true,

  11476.             "sort": 0,

  11477.             "value_type": "individual"

  11478.           },

  11479.           "type": "graph",

  11480.           "xaxis": {

  11481.             "mode": "time",

  11482.             "show": true,

  11483.             "values": []

  11484.           },

  11485.           "yaxes": [

  11486.             {

  11487.               "format": "short",

  11488.               "label": "Number of stale forward extremities in room",

  11489.               "logBase": 1,

  11490.               "min": "0",

  11491.               "show": true

  11492.             },

  11493.             {

  11494.               "format": "short",

  11495.               "logBase": 1,

  11496.               "show": true

  11497.             }

  11498.           ],

  11499.           "yaxis": {

  11500.             "align": false

  11501.           }

  11502.         },

  11503.         {

  11504.           "cards": {

  11505.             "cardPadding": 0

  11506.           },

  11507.           "color": {

  11508.             "cardColor": "#73BF69",

  11509.             "colorScale": "sqrt",

  11510.             "colorScheme": "interpolateInferno",

  11511.             "exponent": 0.5,

  11512.             "min": 0,

  11513.             "mode": "opacity"

  11514.           },

  11515.           "dataFormat": "tsbuckets",

  11516.           "datasource": {

  11517.             "uid": "${DS_PROMETHEUS}"

  11518.           },

  11519.           "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",

  11520.           "fieldConfig": {

  11521.             "defaults": {

  11522.               "custom": {

  11523.                 "hideFrom": {

  11524.                   "legend": false,

  11525.                   "tooltip": false,

  11526.                   "viz": false

  11527.                 },

  11528.                 "scaleDistribution": {

  11529.                   "type": "linear"

  11530.                 }

  11531.               }

  11532.             },

  11533.             "overrides": []

  11534.           },

  11535.           "gridPos": {

  11536.             "h": 8,

  11537.             "w": 12,

  11538.             "x": 0,

  11539.             "y": 66

  11540.           },

  11541.           "heatmap": {},

  11542.           "hideZeroBuckets": true,

  11543.           "highlightCards": true,

  11544.           "id": 131,

  11545.           "legend": {

  11546.             "show": true

  11547.           },

  11548.           "links": [],

  11549.           "options": {

  11550.             "calculate": false,

  11551.             "calculation": {},

  11552.             "cellGap": 1,

  11553.             "cellValues": {},

  11554.             "color": {

  11555.               "exponent": 0.5,

  11556.               "fill": "#73BF69",

  11557.               "min": 0,

  11558.               "mode": "opacity",

  11559.               "reverse": false,

  11560.               "scale": "exponential",

  11561.               "scheme": "Oranges",

  11562.               "steps": 128

  11563.             },

  11564.             "exemplars": {

  11565.               "color": "rgba(255,0,255,0.7)"

  11566.             },

  11567.             "filterValues": {

  11568.               "le": 1e-9

  11569.             },

  11570.             "legend": {

  11571.               "show": true

  11572.             },

  11573.             "rowsFrame": {

  11574.               "layout": "auto"

  11575.             },

  11576.             "showValue": "never",

  11577.             "tooltip": {

  11578.               "show": true,

  11579.               "yHistogram": true

  11580.             },

  11581.             "yAxis": {

  11582.               "axisPlacement": "left",

  11583.               "decimals": 0,

  11584.               "reverse": false,

  11585.               "unit": "short"

  11586.             }

  11587.           },

  11588.           "pluginVersion": "9.2.2",

  11589.           "reverseYBuckets": false,

  11590.           "targets": [

  11591.             {

  11592.               "datasource": {

  11593.                 "uid": "${DS_PROMETHEUS}"

  11594.               },

  11595.               "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",

  11596.               "format": "heatmap",

  11597.               "interval": "",

  11598.               "intervalFactor": 1,

  11599.               "legendFormat": "{{le}}",

  11600.               "refId": "A"

  11601.             }

  11602.           ],

  11603.           "title": "Number of state resolution performed, by number of state groups involved (heatmap)",

  11604.           "tooltip": {

  11605.             "show": true,

  11606.             "showHistogram": true

  11607.           },

  11608.           "type": "heatmap",

  11609.           "xAxis": {

  11610.             "show": true

  11611.           },

  11612.           "yAxis": {

  11613.             "decimals": 0,

  11614.             "format": "short",

  11615.             "logBase": 1,

  11616.             "show": true

  11617.           },

  11618.           "yBucketBound": "auto"

  11619.         },

  11620.         {

  11621.           "aliasColors": {},

  11622.           "bars": false,

  11623.           "dashLength": 10,

  11624.           "dashes": false,

  11625.           "datasource": {

  11626.             "type": "prometheus",

  11627.             "uid": "${DS_PROMETHEUS}"

  11628.           },

  11629.           "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",

  11630.           "fieldConfig": {

  11631.             "defaults": {

  11632.               "links": []

  11633.             },

  11634.             "overrides": []

  11635.           },

  11636.           "fill": 1,

  11637.           "fillGradient": 0,

  11638.           "gridPos": {

  11639.             "h": 8,

  11640.             "w": 12,

  11641.             "x": 12,

  11642.             "y": 66

  11643.           },

  11644.           "hiddenSeries": false,

  11645.           "id": 132,

  11646.           "interval": "",

  11647.           "legend": {

  11648.             "avg": false,

  11649.             "current": false,

  11650.             "max": false,

  11651.             "min": false,

  11652.             "show": true,

  11653.             "total": false,

  11654.             "values": false

  11655.           },

  11656.           "lines": true,

  11657.           "linewidth": 1,

  11658.           "links": [],

  11659.           "nullPointMode": "null",

  11660.           "options": {

  11661.             "alertThreshold": true

  11662.           },

  11663.           "percentage": false,

  11664.           "pluginVersion": "9.2.2",

  11665.           "pointradius": 2,

  11666.           "points": false,

  11667.           "renderer": "flot",

  11668.           "seriesOverrides": [],

  11669.           "spaceLength": 10,

  11670.           "stack": false,

  11671.           "steppedLine": false,

  11672.           "targets": [

  11673.             {

  11674.               "datasource": {

  11675.                 "uid": "${DS_PROMETHEUS}"

  11676.               },

  11677.               "editorMode": "code",

  11678.               "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11679.               "format": "time_series",

  11680.               "interval": "",

  11681.               "intervalFactor": 1,

  11682.               "legendFormat": "50%",

  11683.               "range": true,

  11684.               "refId": "A"

  11685.             },

  11686.             {

  11687.               "datasource": {

  11688.                 "uid": "${DS_PROMETHEUS}"

  11689.               },

  11690.               "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11691.               "format": "time_series",

  11692.               "interval": "",

  11693.               "intervalFactor": 1,

  11694.               "legendFormat": "75%",

  11695.               "refId": "B"

  11696.             },

  11697.             {

  11698.               "datasource": {

  11699.                 "uid": "${DS_PROMETHEUS}"

  11700.               },

  11701.               "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11702.               "format": "time_series",

  11703.               "interval": "",

  11704.               "intervalFactor": 1,

  11705.               "legendFormat": "90%",

  11706.               "refId": "C"

  11707.             },

  11708.             {

  11709.               "datasource": {

  11710.                 "uid": "${DS_PROMETHEUS}"

  11711.               },

  11712.               "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11713.               "format": "time_series",

  11714.               "interval": "",

  11715.               "intervalFactor": 1,

  11716.               "legendFormat": "99%",

  11717.               "refId": "D"

  11718.             }

  11719.           ],

  11720.           "thresholds": [],

  11721.           "timeRegions": [],

  11722.           "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",

  11723.           "tooltip": {

  11724.             "shared": true,

  11725.             "sort": 0,

  11726.             "value_type": "individual"

  11727.           },

  11728.           "type": "graph",

  11729.           "xaxis": {

  11730.             "mode": "time",

  11731.             "show": true,

  11732.             "values": []

  11733.           },

  11734.           "yaxes": [

  11735.             {

  11736.               "format": "short",

  11737.               "label": "Number of state groups",

  11738.               "logBase": 1,

  11739.               "min": "0",

  11740.               "show": true

  11741.             },

  11742.             {

  11743.               "format": "short",

  11744.               "logBase": 1,

  11745.               "show": true

  11746.             }

  11747.           ],

  11748.           "yaxis": {

  11749.             "align": false

  11750.           }

  11751.         },

  11752.         {

  11753.           "aliasColors": {},

  11754.           "bars": false,

  11755.           "dashLength": 10,

  11756.           "dashes": false,

  11757.           "datasource": {

  11758.             "uid": "${DS_PROMETHEUS}"

  11759.           },

  11760.           "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",

  11761.           "fill": 1,

  11762.           "fillGradient": 0,

  11763.           "gridPos": {

  11764.             "h": 8,

  11765.             "w": 12,

  11766.             "x": 0,

  11767.             "y": 74

  11768.           },

  11769.           "hiddenSeries": false,

  11770.           "id": 179,

  11771.           "legend": {

  11772.             "avg": false,

  11773.             "current": false,

  11774.             "max": false,

  11775.             "min": false,

  11776.             "show": true,

  11777.             "total": false,

  11778.             "values": false

  11779.           },

  11780.           "lines": true,

  11781.           "linewidth": 1,

  11782.           "nullPointMode": "null",

  11783.           "options": {

  11784.             "alertThreshold": true

  11785.           },

  11786.           "percentage": false,

  11787.           "pluginVersion": "9.2.2",

  11788.           "pointradius": 2,

  11789.           "points": false,

  11790.           "renderer": "flot",

  11791.           "seriesOverrides": [],

  11792.           "spaceLength": 10,

  11793.           "stack": false,

  11794.           "steppedLine": false,

  11795.           "targets": [

  11796.             {

  11797.               "datasource": {

  11798.                 "uid": "${DS_PROMETHEUS}"

  11799.               },

  11800.               "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11801.               "interval": "",

  11802.               "legendFormat": "State res ",

  11803.               "refId": "A"

  11804.             },

  11805.             {

  11806.               "datasource": {

  11807.                 "uid": "${DS_PROMETHEUS}"

  11808.               },

  11809.               "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11810.               "interval": "",

  11811.               "legendFormat": "Potential to prune",

  11812.               "refId": "B"

  11813.             },

  11814.             {

  11815.               "datasource": {

  11816.                 "uid": "${DS_PROMETHEUS}"

  11817.               },

  11818.               "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",

  11819.               "interval": "",

  11820.               "legendFormat": "Pruned",

  11821.               "refId": "C"

  11822.             }

  11823.           ],

  11824.           "thresholds": [],

  11825.           "timeRegions": [],

  11826.           "title": "Stale extremity dropping",

  11827.           "tooltip": {

  11828.             "shared": true,

  11829.             "sort": 0,

  11830.             "value_type": "individual"

  11831.           },

  11832.           "type": "graph",

  11833.           "xaxis": {

  11834.             "mode": "time",

  11835.             "show": true,

  11836.             "values": []

  11837.           },

  11838.           "yaxes": [

  11839.             {

  11840.               "format": "hertz",

  11841.               "logBase": 1,

  11842.               "show": true

  11843.             },

  11844.             {

  11845.               "format": "short",

  11846.               "logBase": 1,

  11847.               "show": true

  11848.             }

  11849.           ],

  11850.           "yaxis": {

  11851.             "align": false

  11852.           }

  11853.         }

  11854.       ],

  11855.       "targets": [

  11856.         {

  11857.           "datasource": {

  11858.             "type": "prometheus",

  11859.             "uid": "${DS_PROMETHEUS}"

  11860.           },

  11861.           "refId": "A"

  11862.         }

  11863.       ],

  11864.       "title": "Extremities",

  11865.       "type": "row"

  11866.     },

  11867.     {

  11868.       "collapsed": true,

  11869.       "datasource": {

  11870.         "type": "prometheus",

  11871.         "uid": "${DS_PROMETHEUS}"

  11872.       },

  11873.       "gridPos": {

  11874.         "h": 1,

  11875.         "w": 24,

  11876.         "x": 0,

  11877.         "y": 42

  11878.       },

  11879.       "id": 158,

  11880.       "panels": [

  11881.         {

  11882.           "aliasColors": {},

  11883.           "bars": false,

  11884.           "dashLength": 10,

  11885.           "dashes": false,

  11886.           "datasource": {

  11887.             "type": "prometheus",

  11888.             "uid": "${DS_PROMETHEUS}"

  11889.           },

  11890.           "fieldConfig": {

  11891.             "defaults": {

  11892.               "links": []

  11893.             },

  11894.             "overrides": []

  11895.           },

  11896.           "fill": 1,

  11897.           "fillGradient": 0,

  11898.           "gridPos": {

  11899.             "h": 8,

  11900.             "w": 12,

  11901.             "x": 0,

  11902.             "y": 43

  11903.           },

  11904.           "hiddenSeries": false,

  11905.           "id": 156,

  11906.           "legend": {

  11907.             "avg": false,

  11908.             "current": false,

  11909.             "max": false,

  11910.             "min": false,

  11911.             "show": true,

  11912.             "total": false,

  11913.             "values": false

  11914.           },

  11915.           "lines": true,

  11916.           "linewidth": 1,

  11917.           "links": [],

  11918.           "nullPointMode": "null",

  11919.           "options": {

  11920.             "alertThreshold": true

  11921.           },

  11922.           "percentage": false,

  11923.           "pluginVersion": "9.2.2",

  11924.           "pointradius": 5,

  11925.           "points": false,

  11926.           "renderer": "flot",

  11927.           "seriesOverrides": [

  11928.             {

  11929.               "$$hashKey": "object:632",

  11930.               "alias": "Max",

  11931.               "color": "#bf1b00",

  11932.               "fill": 0,

  11933.               "linewidth": 2

  11934.             }

  11935.           ],

  11936.           "spaceLength": 10,

  11937.           "stack": false,

  11938.           "steppedLine": false,

  11939.           "targets": [

  11940.             {

  11941.               "datasource": {

  11942.                 "uid": "${DS_PROMETHEUS}"

  11943.               },

  11944.               "editorMode": "code",

  11945.               "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})",

  11946.               "format": "time_series",

  11947.               "interval": "",

  11948.               "intervalFactor": 1,

  11949.               "legendFormat": "Max",

  11950.               "range": true,

  11951.               "refId": "B"

  11952.             },

  11953.             {

  11954.               "datasource": {

  11955.                 "type": "prometheus",

  11956.                 "uid": "${DS_PROMETHEUS}"

  11957.               },

  11958.               "editorMode": "code",

  11959.               "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})",

  11960.               "hide": false,

  11961.               "legendFormat": "Current",

  11962.               "range": true,

  11963.               "refId": "C"

  11964.             }

  11965.           ],

  11966.           "thresholds": [],

  11967.           "timeRegions": [],

  11968.           "title": "MAU Limits",

  11969.           "tooltip": {

  11970.             "shared": true,

  11971.             "sort": 0,

  11972.             "value_type": "individual"

  11973.           },

  11974.           "type": "graph",

  11975.           "xaxis": {

  11976.             "mode": "time",

  11977.             "show": true,

  11978.             "values": []

  11979.           },

  11980.           "yaxes": [

  11981.             {

  11982.               "$$hashKey": "object:176",

  11983.               "format": "short",

  11984.               "logBase": 1,

  11985.               "min": "0",

  11986.               "show": true

  11987.             },

  11988.             {

  11989.               "$$hashKey": "object:177",

  11990.               "format": "short",

  11991.               "logBase": 1,

  11992.               "show": true

  11993.             }

  11994.           ],

  11995.           "yaxis": {

  11996.             "align": false

  11997.           }

  11998.         },

  11999.         {

  12000.           "aliasColors": {},

  12001.           "bars": false,

  12002.           "dashLength": 10,

  12003.           "dashes": false,

  12004.           "datasource": {

  12005.             "uid": "${DS_PROMETHEUS}"

  12006.           },

  12007.           "fill": 1,

  12008.           "fillGradient": 0,

  12009.           "gridPos": {

  12010.             "h": 8,

  12011.             "w": 12,

  12012.             "x": 12,

  12013.             "y": 43

  12014.           },

  12015.           "hiddenSeries": false,

  12016.           "id": 160,

  12017.           "legend": {

  12018.             "avg": false,

  12019.             "current": false,

  12020.             "max": false,

  12021.             "min": false,

  12022.             "show": true,

  12023.             "total": false,

  12024.             "values": false

  12025.           },

  12026.           "lines": true,

  12027.           "linewidth": 1,

  12028.           "nullPointMode": "null",

  12029.           "options": {

  12030.             "alertThreshold": true

  12031.           },

  12032.           "percentage": false,

  12033.           "pluginVersion": "9.2.2",

  12034.           "pointradius": 2,

  12035.           "points": false,

  12036.           "renderer": "flot",

  12037.           "seriesOverrides": [],

  12038.           "spaceLength": 10,

  12039.           "stack": false,

  12040.           "steppedLine": false,

  12041.           "targets": [

  12042.             {

  12043.               "datasource": {

  12044.                 "uid": "${DS_PROMETHEUS}"

  12045.               },

  12046.               "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",

  12047.               "interval": "",

  12048.               "legendFormat": "{{ app_service }}",

  12049.               "refId": "A"

  12050.             }

  12051.           ],

  12052.           "thresholds": [],

  12053.           "timeRegions": [],

  12054.           "title": "MAU by Appservice",

  12055.           "tooltip": {

  12056.             "shared": true,

  12057.             "sort": 0,

  12058.             "value_type": "individual"

  12059.           },

  12060.           "type": "graph",

  12061.           "xaxis": {

  12062.             "mode": "time",

  12063.             "show": true,

  12064.             "values": []

  12065.           },

  12066.           "yaxes": [

  12067.             {

  12068.               "format": "short",

  12069.               "logBase": 1,

  12070.               "show": true

  12071.             },

  12072.             {

  12073.               "format": "short",

  12074.               "logBase": 1,

  12075.               "show": true

  12076.             }

  12077.           ],

  12078.           "yaxis": {

  12079.             "align": false

  12080.           }

  12081.         }

  12082.       ],

  12083.       "targets": [

  12084.         {

  12085.           "datasource": {

  12086.             "type": "prometheus",

  12087.             "uid": "${DS_PROMETHEUS}"

  12088.           },

  12089.           "refId": "A"

  12090.         }

  12091.       ],

  12092.       "title": "MAU",

  12093.       "type": "row"

  12094.     },

  12095.     {

  12096.       "collapsed": true,

  12097.       "datasource": {

  12098.         "type": "prometheus",

  12099.         "uid": "${DS_PROMETHEUS}"

  12100.       },

  12101.       "gridPos": {

  12102.         "h": 1,

  12103.         "w": 24,

  12104.         "x": 0,

  12105.         "y": 43

  12106.       },

  12107.       "id": 177,

  12108.       "panels": [

  12109.         {

  12110.           "aliasColors": {},

  12111.           "bars": false,

  12112.           "dashLength": 10,

  12113.           "dashes": false,

  12114.           "datasource": {

  12115.             "type": "prometheus",

  12116.             "uid": "${DS_PROMETHEUS}"

  12117.           },

  12118.           "fieldConfig": {

  12119.             "defaults": {

  12120.               "links": []

  12121.             },

  12122.             "overrides": []

  12123.           },

  12124.           "fill": 1,

  12125.           "fillGradient": 0,

  12126.           "gridPos": {

  12127.             "h": 7,

  12128.             "w": 12,

  12129.             "x": 0,

  12130.             "y": 44

  12131.           },

  12132.           "hiddenSeries": false,

  12133.           "id": 173,

  12134.           "legend": {

  12135.             "avg": false,

  12136.             "current": false,

  12137.             "max": false,

  12138.             "min": false,

  12139.             "show": true,

  12140.             "total": false,

  12141.             "values": false

  12142.           },

  12143.           "lines": true,

  12144.           "linewidth": 1,

  12145.           "links": [],

  12146.           "nullPointMode": "null",

  12147.           "options": {

  12148.             "alertThreshold": true

  12149.           },

  12150.           "percentage": false,

  12151.           "pluginVersion": "9.2.2",

  12152.           "pointradius": 5,

  12153.           "points": false,

  12154.           "renderer": "flot",

  12155.           "seriesOverrides": [],

  12156.           "spaceLength": 10,

  12157.           "stack": false,

  12158.           "steppedLine": false,

  12159.           "targets": [

  12160.             {

  12161.               "datasource": {

  12162.                 "uid": "${DS_PROMETHEUS}"

  12163.               },

  12164.               "editorMode": "code",

  12165.               "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12166.               "format": "time_series",

  12167.               "hide": false,

  12168.               "intervalFactor": 2,

  12169.               "legendFormat": "{{stream}} {{index}}",

  12170.               "metric": "synapse_notifier",

  12171.               "range": true,

  12172.               "refId": "A",

  12173.               "step": 2

  12174.             }

  12175.           ],

  12176.           "thresholds": [],

  12177.           "timeRegions": [],

  12178.           "title": "Notifier Streams Woken",

  12179.           "tooltip": {

  12180.             "shared": true,

  12181.             "sort": 2,

  12182.             "value_type": "individual"

  12183.           },

  12184.           "type": "graph",

  12185.           "xaxis": {

  12186.             "mode": "time",

  12187.             "show": true,

  12188.             "values": []

  12189.           },

  12190.           "yaxes": [

  12191.             {

  12192.               "$$hashKey": "object:734",

  12193.               "format": "hertz",

  12194.               "logBase": 1,

  12195.               "show": true

  12196.             },

  12197.             {

  12198.               "$$hashKey": "object:735",

  12199.               "format": "short",

  12200.               "logBase": 1,

  12201.               "show": true

  12202.             }

  12203.           ],

  12204.           "yaxis": {

  12205.             "align": false

  12206.           }

  12207.         },

  12208.         {

  12209.           "aliasColors": {},

  12210.           "bars": false,

  12211.           "dashLength": 10,

  12212.           "dashes": false,

  12213.           "datasource": {

  12214.             "type": "prometheus",

  12215.             "uid": "${DS_PROMETHEUS}"

  12216.           },

  12217.           "fieldConfig": {

  12218.             "defaults": {

  12219.               "links": []

  12220.             },

  12221.             "overrides": []

  12222.           },

  12223.           "fill": 1,

  12224.           "fillGradient": 0,

  12225.           "gridPos": {

  12226.             "h": 7,

  12227.             "w": 12,

  12228.             "x": 12,

  12229.             "y": 44

  12230.           },

  12231.           "hiddenSeries": false,

  12232.           "id": 175,

  12233.           "legend": {

  12234.             "avg": false,

  12235.             "current": false,

  12236.             "max": false,

  12237.             "min": false,

  12238.             "show": true,

  12239.             "total": false,

  12240.             "values": false

  12241.           },

  12242.           "lines": true,

  12243.           "linewidth": 1,

  12244.           "links": [],

  12245.           "nullPointMode": "null",

  12246.           "options": {

  12247.             "alertThreshold": true

  12248.           },

  12249.           "percentage": false,

  12250.           "pluginVersion": "9.2.2",

  12251.           "pointradius": 5,

  12252.           "points": false,

  12253.           "renderer": "flot",

  12254.           "seriesOverrides": [],

  12255.           "spaceLength": 10,

  12256.           "stack": false,

  12257.           "steppedLine": false,

  12258.           "targets": [

  12259.             {

  12260.               "datasource": {

  12261.                 "uid": "${DS_PROMETHEUS}"

  12262.               },

  12263.               "editorMode": "code",

  12264.               "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",

  12265.               "format": "time_series",

  12266.               "interval": "",

  12267.               "intervalFactor": 2,

  12268.               "legendFormat": "{{type}} {{index}}",

  12269.               "range": true,

  12270.               "refId": "A",

  12271.               "step": 2

  12272.             }

  12273.           ],

  12274.           "thresholds": [],

  12275.           "timeRegions": [],

  12276.           "title": "Presence Stream Fetch Type Rates",

  12277.           "tooltip": {

  12278.             "shared": true,

  12279.             "sort": 2,

  12280.             "value_type": "individual"

  12281.           },

  12282.           "type": "graph",

  12283.           "xaxis": {

  12284.             "mode": "time",

  12285.             "show": true,

  12286.             "values": []

  12287.           },

  12288.           "yaxes": [

  12289.             {

  12290.               "$$hashKey": "object:819",

  12291.               "format": "hertz",

  12292.               "logBase": 1,

  12293.               "min": "0",

  12294.               "show": true

  12295.             },

  12296.             {

  12297.               "$$hashKey": "object:820",

  12298.               "format": "short",

  12299.               "logBase": 1,

  12300.               "show": true

  12301.             }

  12302.           ],

  12303.           "yaxis": {

  12304.             "align": false

  12305.           }

  12306.         }

  12307.       ],

  12308.       "targets": [

  12309.         {

  12310.           "datasource": {

  12311.             "type": "prometheus",

  12312.             "uid": "${DS_PROMETHEUS}"

  12313.           },

  12314.           "refId": "A"

  12315.         }

  12316.       ],

  12317.       "title": "Notifier",

  12318.       "type": "row"

  12319.     },

  12320.     {

  12321.       "collapsed": true,

  12322.       "datasource": {

  12323.         "type": "prometheus",

  12324.         "uid": "${DS_PROMETHEUS}"

  12325.       },

  12326.       "gridPos": {

  12327.         "h": 1,

  12328.         "w": 24,

  12329.         "x": 0,

  12330.         "y": 44

  12331.       },

  12332.       "id": 170,

  12333.       "panels": [

  12334.         {

  12335.           "aliasColors": {},

  12336.           "bars": false,

  12337.           "dashLength": 10,

  12338.           "dashes": false,

  12339.           "datasource": {

  12340.             "type": "prometheus",

  12341.             "uid": "${DS_PROMETHEUS}"

  12342.           },

  12343.           "fill": 1,

  12344.           "fillGradient": 0,

  12345.           "gridPos": {

  12346.             "h": 8,

  12347.             "w": 12,

  12348.             "x": 0,

  12349.             "y": 45

  12350.           },

  12351.           "hiddenSeries": false,

  12352.           "id": 168,

  12353.           "legend": {

  12354.             "avg": false,

  12355.             "current": false,

  12356.             "max": false,

  12357.             "min": false,

  12358.             "show": true,

  12359.             "total": false,

  12360.             "values": false

  12361.           },

  12362.           "lines": true,

  12363.           "linewidth": 1,

  12364.           "nullPointMode": "null",

  12365.           "options": {

  12366.             "alertThreshold": true

  12367.           },

  12368.           "percentage": false,

  12369.           "pluginVersion": "9.2.2",

  12370.           "pointradius": 2,

  12371.           "points": false,

  12372.           "renderer": "flot",

  12373.           "seriesOverrides": [],

  12374.           "spaceLength": 10,

  12375.           "stack": false,

  12376.           "steppedLine": false,

  12377.           "targets": [

  12378.             {

  12379.               "datasource": {

  12380.                 "uid": "${DS_PROMETHEUS}"

  12381.               },

  12382.               "editorMode": "code",

  12383.               "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])",

  12384.               "interval": "",

  12385.               "legendFormat": "{{service}}",

  12386.               "range": true,

  12387.               "refId": "A"

  12388.             }

  12389.           ],

  12390.           "thresholds": [],

  12391.           "timeRegions": [],

  12392.           "title": "Sent Events rate",

  12393.           "tooltip": {

  12394.             "shared": true,

  12395.             "sort": 0,

  12396.             "value_type": "individual"

  12397.           },

  12398.           "type": "graph",

  12399.           "xaxis": {

  12400.             "mode": "time",

  12401.             "show": true,

  12402.             "values": []

  12403.           },

  12404.           "yaxes": [

  12405.             {

  12406.               "$$hashKey": "object:177",

  12407.               "format": "hertz",

  12408.               "logBase": 1,

  12409.               "show": true

  12410.             },

  12411.             {

  12412.               "$$hashKey": "object:178",

  12413.               "format": "short",

  12414.               "logBase": 1,

  12415.               "show": true

  12416.             }

  12417.           ],

  12418.           "yaxis": {

  12419.             "align": false

  12420.           }

  12421.         },

  12422.         {

  12423.           "aliasColors": {},

  12424.           "bars": false,

  12425.           "dashLength": 10,

  12426.           "dashes": false,

  12427.           "datasource": {

  12428.             "type": "prometheus",

  12429.             "uid": "${DS_PROMETHEUS}"

  12430.           },

  12431.           "fill": 1,

  12432.           "fillGradient": 0,

  12433.           "gridPos": {

  12434.             "h": 8,

  12435.             "w": 12,

  12436.             "x": 12,

  12437.             "y": 45

  12438.           },

  12439.           "hiddenSeries": false,

  12440.           "id": 171,

  12441.           "legend": {

  12442.             "avg": false,

  12443.             "current": false,

  12444.             "max": false,

  12445.             "min": false,

  12446.             "show": true,

  12447.             "total": false,

  12448.             "values": false

  12449.           },

  12450.           "lines": true,

  12451.           "linewidth": 1,

  12452.           "nullPointMode": "null",

  12453.           "options": {

  12454.             "alertThreshold": true

  12455.           },

  12456.           "percentage": false,

  12457.           "pluginVersion": "9.2.2",

  12458.           "pointradius": 2,

  12459.           "points": false,

  12460.           "renderer": "flot",

  12461.           "seriesOverrides": [],

  12462.           "spaceLength": 10,

  12463.           "stack": false,

  12464.           "steppedLine": false,

  12465.           "targets": [

  12466.             {

  12467.               "datasource": {

  12468.                 "uid": "${DS_PROMETHEUS}"

  12469.               },

  12470.               "editorMode": "code",

  12471.               "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])",

  12472.               "interval": "",

  12473.               "legendFormat": "{{exported_service }} {{ service }}",

  12474.               "range": true,

  12475.               "refId": "A"

  12476.             }

  12477.           ],

  12478.           "thresholds": [],

  12479.           "timeRegions": [],

  12480.           "title": "Transactions rate",

  12481.           "tooltip": {

  12482.             "shared": true,

  12483.             "sort": 0,

  12484.             "value_type": "individual"

  12485.           },

  12486.           "type": "graph",

  12487.           "xaxis": {

  12488.             "mode": "time",

  12489.             "show": true,

  12490.             "values": []

  12491.           },

  12492.           "yaxes": [

  12493.             {

  12494.               "$$hashKey": "object:260",

  12495.               "format": "hertz",

  12496.               "logBase": 1,

  12497.               "show": true

  12498.             },

  12499.             {

  12500.               "$$hashKey": "object:261",

  12501.               "format": "short",

  12502.               "logBase": 1,

  12503.               "show": true

  12504.             }

  12505.           ],

  12506.           "yaxis": {

  12507.             "align": false

  12508.           }

  12509.         }

  12510.       ],

  12511.       "targets": [

  12512.         {

  12513.           "datasource": {

  12514.             "type": "prometheus",

  12515.             "uid": "${DS_PROMETHEUS}"

  12516.           },

  12517.           "refId": "A"

  12518.         }

  12519.       ],

  12520.       "title": "Appservices",

  12521.       "type": "row"

  12522.     },

  12523.     {

  12524.       "collapsed": true,

  12525.       "datasource": {

  12526.         "type": "prometheus",

  12527.         "uid": "${DS_PROMETHEUS}"

  12528.       },

  12529.       "gridPos": {

  12530.         "h": 1,

  12531.         "w": 24,

  12532.         "x": 0,

  12533.         "y": 45

  12534.       },

  12535.       "id": 188,

  12536.       "panels": [

  12537.         {

  12538.           "aliasColors": {},

  12539.           "bars": false,

  12540.           "dashLength": 10,

  12541.           "dashes": false,

  12542.           "datasource": {

  12543.             "uid": "${DS_PROMETHEUS}"

  12544.           },

  12545.           "fill": 1,

  12546.           "fillGradient": 0,

  12547.           "gridPos": {

  12548.             "h": 8,

  12549.             "w": 12,

  12550.             "x": 0,

  12551.             "y": 46

  12552.           },

  12553.           "hiddenSeries": false,

  12554.           "id": 182,

  12555.           "legend": {

  12556.             "avg": false,

  12557.             "current": false,

  12558.             "max": false,

  12559.             "min": false,

  12560.             "show": true,

  12561.             "total": false,

  12562.             "values": false

  12563.           },

  12564.           "lines": true,

  12565.           "linewidth": 1,

  12566.           "nullPointMode": "null",

  12567.           "options": {

  12568.             "alertThreshold": true

  12569.           },

  12570.           "percentage": false,

  12571.           "pluginVersion": "9.2.2",

  12572.           "pointradius": 2,

  12573.           "points": false,

  12574.           "renderer": "flot",

  12575.           "seriesOverrides": [],

  12576.           "spaceLength": 10,

  12577.           "stack": false,

  12578.           "steppedLine": false,

  12579.           "targets": [

  12580.             {

  12581.               "datasource": {

  12582.                 "uid": "${DS_PROMETHEUS}"

  12583.               },

  12584.               "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12585.               "interval": "",

  12586.               "legendFormat": "Notified",

  12587.               "refId": "A"

  12588.             },

  12589.             {

  12590.               "datasource": {

  12591.                 "uid": "${DS_PROMETHEUS}"

  12592.               },

  12593.               "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12594.               "interval": "",

  12595.               "legendFormat": "Remote ping",

  12596.               "refId": "B"

  12597.             },

  12598.             {

  12599.               "datasource": {

  12600.                 "uid": "${DS_PROMETHEUS}"

  12601.               },

  12602.               "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12603.               "interval": "",

  12604.               "legendFormat": "Total updates",

  12605.               "refId": "C"

  12606.             },

  12607.             {

  12608.               "datasource": {

  12609.                 "uid": "${DS_PROMETHEUS}"

  12610.               },

  12611.               "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12612.               "interval": "",

  12613.               "legendFormat": "Remote updates",

  12614.               "refId": "D"

  12615.             },

  12616.             {

  12617.               "datasource": {

  12618.                 "uid": "${DS_PROMETHEUS}"

  12619.               },

  12620.               "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12621.               "interval": "",

  12622.               "legendFormat": "Bump active time",

  12623.               "refId": "E"

  12624.             }

  12625.           ],

  12626.           "thresholds": [],

  12627.           "timeRegions": [],

  12628.           "title": "Presence",

  12629.           "tooltip": {

  12630.             "shared": true,

  12631.             "sort": 2,

  12632.             "value_type": "individual"

  12633.           },

  12634.           "type": "graph",

  12635.           "xaxis": {

  12636.             "mode": "time",

  12637.             "show": true,

  12638.             "values": []

  12639.           },

  12640.           "yaxes": [

  12641.             {

  12642.               "format": "hertz",

  12643.               "logBase": 1,

  12644.               "show": true

  12645.             },

  12646.             {

  12647.               "format": "short",

  12648.               "logBase": 1,

  12649.               "show": true

  12650.             }

  12651.           ],

  12652.           "yaxis": {

  12653.             "align": false

  12654.           }

  12655.         },

  12656.         {

  12657.           "aliasColors": {},

  12658.           "bars": false,

  12659.           "dashLength": 10,

  12660.           "dashes": false,

  12661.           "datasource": {

  12662.             "type": "prometheus",

  12663.             "uid": "${DS_PROMETHEUS}"

  12664.           },

  12665.           "fill": 1,

  12666.           "fillGradient": 0,

  12667.           "gridPos": {

  12668.             "h": 8,

  12669.             "w": 12,

  12670.             "x": 12,

  12671.             "y": 46

  12672.           },

  12673.           "hiddenSeries": false,

  12674.           "id": 184,

  12675.           "legend": {

  12676.             "avg": false,

  12677.             "current": false,

  12678.             "max": false,

  12679.             "min": false,

  12680.             "show": true,

  12681.             "total": false,

  12682.             "values": false

  12683.           },

  12684.           "lines": true,

  12685.           "linewidth": 1,

  12686.           "nullPointMode": "null",

  12687.           "options": {

  12688.             "alertThreshold": true

  12689.           },

  12690.           "percentage": false,

  12691.           "pluginVersion": "9.2.2",

  12692.           "pointradius": 2,

  12693.           "points": false,

  12694.           "renderer": "flot",

  12695.           "seriesOverrides": [],

  12696.           "spaceLength": 10,

  12697.           "stack": false,

  12698.           "steppedLine": false,

  12699.           "targets": [

  12700.             {

  12701.               "datasource": {

  12702.                 "uid": "${DS_PROMETHEUS}"

  12703.               },

  12704.               "editorMode": "code",

  12705.               "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12706.               "interval": "",

  12707.               "legendFormat": "{{from}} -> {{to}}",

  12708.               "range": true,

  12709.               "refId": "A"

  12710.             }

  12711.           ],

  12712.           "thresholds": [],

  12713.           "timeRegions": [],

  12714.           "title": "Presence state transitions",

  12715.           "tooltip": {

  12716.             "shared": true,

  12717.             "sort": 2,

  12718.             "value_type": "individual"

  12719.           },

  12720.           "type": "graph",

  12721.           "xaxis": {

  12722.             "mode": "time",

  12723.             "show": true,

  12724.             "values": []

  12725.           },

  12726.           "yaxes": [

  12727.             {

  12728.               "$$hashKey": "object:1090",

  12729.               "format": "hertz",

  12730.               "logBase": 1,

  12731.               "show": true

  12732.             },

  12733.             {

  12734.               "$$hashKey": "object:1091",

  12735.               "format": "short",

  12736.               "logBase": 1,

  12737.               "show": true

  12738.             }

  12739.           ],

  12740.           "yaxis": {

  12741.             "align": false

  12742.           }

  12743.         },

  12744.         {

  12745.           "aliasColors": {},

  12746.           "bars": false,

  12747.           "dashLength": 10,

  12748.           "dashes": false,

  12749.           "datasource": {

  12750.             "type": "prometheus",

  12751.             "uid": "${DS_PROMETHEUS}"

  12752.           },

  12753.           "fill": 1,

  12754.           "fillGradient": 0,

  12755.           "gridPos": {

  12756.             "h": 8,

  12757.             "w": 12,

  12758.             "x": 0,

  12759.             "y": 54

  12760.           },

  12761.           "hiddenSeries": false,

  12762.           "id": 186,

  12763.           "legend": {

  12764.             "avg": false,

  12765.             "current": false,

  12766.             "max": false,

  12767.             "min": false,

  12768.             "show": true,

  12769.             "total": false,

  12770.             "values": false

  12771.           },

  12772.           "lines": true,

  12773.           "linewidth": 1,

  12774.           "nullPointMode": "null",

  12775.           "options": {

  12776.             "alertThreshold": true

  12777.           },

  12778.           "percentage": false,

  12779.           "pluginVersion": "9.2.2",

  12780.           "pointradius": 2,

  12781.           "points": false,

  12782.           "renderer": "flot",

  12783.           "seriesOverrides": [],

  12784.           "spaceLength": 10,

  12785.           "stack": false,

  12786.           "steppedLine": false,

  12787.           "targets": [

  12788.             {

  12789.               "datasource": {

  12790.                 "uid": "${DS_PROMETHEUS}"

  12791.               },

  12792.               "editorMode": "code",

  12793.               "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",

  12794.               "interval": "",

  12795.               "legendFormat": "{{reason}}",

  12796.               "range": true,

  12797.               "refId": "A"

  12798.             }

  12799.           ],

  12800.           "thresholds": [],

  12801.           "timeRegions": [],

  12802.           "title": "Presence  notify reason",

  12803.           "tooltip": {

  12804.             "shared": true,

  12805.             "sort": 2,

  12806.             "value_type": "individual"

  12807.           },

  12808.           "type": "graph",

  12809.           "xaxis": {

  12810.             "mode": "time",

  12811.             "show": true,

  12812.             "values": []

  12813.           },

  12814.           "yaxes": [

  12815.             {

  12816.               "$$hashKey": "object:165",

  12817.               "format": "hertz",

  12818.               "logBase": 1,

  12819.               "show": true

  12820.             },

  12821.             {

  12822.               "$$hashKey": "object:166",

  12823.               "format": "short",

  12824.               "logBase": 1,

  12825.               "show": true

  12826.             }

  12827.           ],

  12828.           "yaxis": {

  12829.             "align": false

  12830.           }

  12831.         }

  12832.       ],

  12833.       "targets": [

  12834.         {

  12835.           "datasource": {

  12836.             "type": "prometheus",

  12837.             "uid": "${DS_PROMETHEUS}"

  12838.           },

  12839.           "refId": "A"

  12840.         }

  12841.       ],

  12842.       "title": "Presence",

  12843.       "type": "row"

  12844.     },

  12845.     {

  12846.       "collapsed": true,

  12847.       "datasource": {

  12848.         "type": "prometheus",

  12849.         "uid": "${DS_PROMETHEUS}"

  12850.       },

  12851.       "gridPos": {

  12852.         "h": 1,

  12853.         "w": 24,

  12854.         "x": 0,

  12855.         "y": 46

  12856.       },

  12857.       "id": 197,

  12858.       "panels": [

  12859.         {

  12860.           "datasource": {

  12861.             "type": "prometheus",

  12862.             "uid": "${DS_PROMETHEUS}"

  12863.           },

  12864.           "fieldConfig": {

  12865.             "defaults": {

  12866.               "color": {

  12867.                 "mode": "palette-classic"

  12868.               },

  12869.               "custom": {

  12870.                 "axisCenteredZero": false,

  12871.                 "axisColorMode": "text",

  12872.                 "axisLabel": "",

  12873.                 "axisPlacement": "auto",

  12874.                 "barAlignment": 0,

  12875.                 "drawStyle": "line",

  12876.                 "fillOpacity": 10,

  12877.                 "gradientMode": "none",

  12878.                 "hideFrom": {

  12879.                   "legend": false,

  12880.                   "tooltip": false,

  12881.                   "viz": false

  12882.                 },

  12883.                 "lineInterpolation": "linear",

  12884.                 "lineWidth": 1,

  12885.                 "pointSize": 5,

  12886.                 "scaleDistribution": {

  12887.                   "type": "linear"

  12888.                 },

  12889.                 "showPoints": "never",

  12890.                 "spanNulls": false,

  12891.                 "stacking": {

  12892.                   "group": "A",

  12893.                   "mode": "none"

  12894.                 },

  12895.                 "thresholdsStyle": {

  12896.                   "mode": "off"

  12897.                 }

  12898.               },

  12899.               "mappings": [],

  12900.               "thresholds": {

  12901.                 "mode": "absolute",

  12902.                 "steps": [

  12903.                   {

  12904.                     "color": "green"

  12905.                   },

  12906.                   {

  12907.                     "color": "red",

  12908.                     "value": 80

  12909.                   }

  12910.                 ]

  12911.               },

  12912.               "unit": "hertz"

  12913.             },

  12914.             "overrides": []

  12915.           },

  12916.           "gridPos": {

  12917.             "h": 8,

  12918.             "w": 12,

  12919.             "x": 0,

  12920.             "y": 47

  12921.           },

  12922.           "id": 191,

  12923.           "options": {

  12924.             "legend": {

  12925.               "calcs": [],

  12926.               "displayMode": "list",

  12927.               "placement": "bottom",

  12928.               "showLegend": true

  12929.             },

  12930.             "tooltip": {

  12931.               "mode": "multi",

  12932.               "sort": "desc"

  12933.             }

  12934.           },

  12935.           "pluginVersion": "9.0.4",

  12936.           "targets": [

  12937.             {

  12938.               "datasource": {

  12939.                 "uid": "${DS_PROMETHEUS}"

  12940.               },

  12941.               "editorMode": "code",

  12942.               "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",

  12943.               "interval": "",

  12944.               "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",

  12945.               "range": true,

  12946.               "refId": "A"

  12947.             }

  12948.           ],

  12949.           "title": "External Cache Set Rate",

  12950.           "type": "timeseries"

  12951.         },

  12952.         {

  12953.           "aliasColors": {},

  12954.           "bars": false,

  12955.           "dashLength": 10,

  12956.           "dashes": false,

  12957.           "datasource": {

  12958.             "type": "prometheus",

  12959.             "uid": "${DS_PROMETHEUS}"

  12960.           },

  12961.           "description": "",

  12962.           "fill": 1,

  12963.           "fillGradient": 0,

  12964.           "gridPos": {

  12965.             "h": 8,

  12966.             "w": 12,

  12967.             "x": 12,

  12968.             "y": 47

  12969.           },

  12970.           "hiddenSeries": false,

  12971.           "id": 193,

  12972.           "legend": {

  12973.             "avg": false,

  12974.             "current": false,

  12975.             "max": false,

  12976.             "min": false,

  12977.             "show": true,

  12978.             "total": false,

  12979.             "values": false

  12980.           },

  12981.           "lines": true,

  12982.           "linewidth": 1,

  12983.           "nullPointMode": "null",

  12984.           "options": {

  12985.             "alertThreshold": true

  12986.           },

  12987.           "percentage": false,

  12988.           "pluginVersion": "9.2.2",

  12989.           "pointradius": 2,

  12990.           "points": false,

  12991.           "renderer": "flot",

  12992.           "seriesOverrides": [],

  12993.           "spaceLength": 10,

  12994.           "stack": false,

  12995.           "steppedLine": false,

  12996.           "targets": [

  12997.             {

  12998.               "datasource": {

  12999.                 "uid": "${DS_PROMETHEUS}"

  13000.               },

  13001.               "editorMode": "code",

  13002.               "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))",

  13003.               "interval": "",

  13004.               "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",

  13005.               "range": true,

  13006.               "refId": "A"

  13007.             }

  13008.           ],

  13009.           "thresholds": [],

  13010.           "timeRegions": [],

  13011.           "title": "External Cache Get Rate",

  13012.           "tooltip": {

  13013.             "shared": true,

  13014.             "sort": 2,

  13015.             "value_type": "individual"

  13016.           },

  13017.           "type": "graph",

  13018.           "xaxis": {

  13019.             "mode": "time",

  13020.             "show": true,

  13021.             "values": []

  13022.           },

  13023.           "yaxes": [

  13024.             {

  13025.               "$$hashKey": "object:390",

  13026.               "format": "hertz",

  13027.               "logBase": 1,

  13028.               "show": true

  13029.             },

  13030.             {

  13031.               "$$hashKey": "object:391",

  13032.               "format": "short",

  13033.               "logBase": 1,

  13034.               "show": true

  13035.             }

  13036.           ],

  13037.           "yaxis": {

  13038.             "align": false

  13039.           }

  13040.         },

  13041.         {

  13042.           "cards": {

  13043.             "cardPadding": -1

  13044.           },

  13045.           "color": {

  13046.             "cardColor": "#b4ff00",

  13047.             "colorScale": "sqrt",

  13048.             "colorScheme": "interpolateInferno",

  13049.             "exponent": 0.5,

  13050.             "min": 0,

  13051.             "mode": "spectrum"

  13052.           },

  13053.           "dataFormat": "tsbuckets",

  13054.           "datasource": {

  13055.             "type": "prometheus",

  13056.             "uid": "${DS_PROMETHEUS}"

  13057.           },

  13058.           "fieldConfig": {

  13059.             "defaults": {

  13060.               "custom": {

  13061.                 "hideFrom": {

  13062.                   "legend": false,

  13063.                   "tooltip": false,

  13064.                   "viz": false

  13065.                 },

  13066.                 "scaleDistribution": {

  13067.                   "type": "linear"

  13068.                 }

  13069.               }

  13070.             },

  13071.             "overrides": []

  13072.           },

  13073.           "gridPos": {

  13074.             "h": 8,

  13075.             "w": 12,

  13076.             "x": 0,

  13077.             "y": 55

  13078.           },

  13079.           "heatmap": {},

  13080.           "hideZeroBuckets": false,

  13081.           "highlightCards": true,

  13082.           "id": 195,

  13083.           "legend": {

  13084.             "show": false

  13085.           },

  13086.           "links": [],

  13087.           "options": {

  13088.             "calculate": false,

  13089.             "calculation": {},

  13090.             "cellGap": -1,

  13091.             "cellValues": {

  13092.               "decimals": 2

  13093.             },

  13094.             "color": {

  13095.               "exponent": 0.5,

  13096.               "fill": "#b4ff00",

  13097.               "min": 0,

  13098.               "mode": "scheme",

  13099.               "reverse": false,

  13100.               "scale": "exponential",

  13101.               "scheme": "Inferno",

  13102.               "steps": 128

  13103.             },

  13104.             "exemplars": {

  13105.               "color": "rgba(255,0,255,0.7)"

  13106.             },

  13107.             "filterValues": {

  13108.               "le": 1e-9

  13109.             },

  13110.             "legend": {

  13111.               "show": false

  13112.             },

  13113.             "rowsFrame": {

  13114.               "layout": "auto"

  13115.             },

  13116.             "showValue": "never",

  13117.             "tooltip": {

  13118.               "show": true,

  13119.               "yHistogram": true

  13120.             },

  13121.             "yAxis": {

  13122.               "axisPlacement": "left",

  13123.               "decimals": 0,

  13124.               "reverse": false,

  13125.               "unit": "s"

  13126.             }

  13127.           },

  13128.           "pluginVersion": "9.2.2",

  13129.           "reverseYBuckets": false,

  13130.           "targets": [

  13131.             {

  13132.               "datasource": {

  13133.                 "uid": "${DS_PROMETHEUS}"

  13134.               },

  13135.               "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)",

  13136.               "format": "heatmap",

  13137.               "instant": false,

  13138.               "interval": "",

  13139.               "intervalFactor": 1,

  13140.               "legendFormat": "{{le}}",

  13141.               "refId": "A"

  13142.             }

  13143.           ],

  13144.           "title": "External Cache Response Time",

  13145.           "tooltip": {

  13146.             "show": true,

  13147.             "showHistogram": true

  13148.           },

  13149.           "tooltipDecimals": 2,

  13150.           "type": "heatmap",

  13151.           "xAxis": {

  13152.             "show": true

  13153.           },

  13154.           "yAxis": {

  13155.             "decimals": 0,

  13156.             "format": "s",

  13157.             "logBase": 1,

  13158.             "show": true

  13159.           },

  13160.           "yBucketBound": "auto"

  13161.         },

  13162.         {

  13163.           "datasource": {

  13164.             "type": "prometheus",

  13165.             "uid": "${DS_PROMETHEUS}"

  13166.           },

  13167.           "description": "",

  13168.           "fieldConfig": {

  13169.             "defaults": {

  13170.               "color": {

  13171.                 "mode": "palette-classic"

  13172.               },

  13173.               "custom": {

  13174.                 "axisCenteredZero": false,

  13175.                 "axisColorMode": "text",

  13176.                 "axisLabel": "",

  13177.                 "axisPlacement": "auto",

  13178.                 "barAlignment": 0,

  13179.                 "drawStyle": "line",

  13180.                 "fillOpacity": 10,

  13181.                 "gradientMode": "none",

  13182.                 "hideFrom": {

  13183.                   "legend": false,

  13184.                   "tooltip": false,

  13185.                   "viz": false

  13186.                 },

  13187.                 "lineInterpolation": "linear",

  13188.                 "lineWidth": 1,

  13189.                 "pointSize": 5,

  13190.                 "scaleDistribution": {

  13191.                   "type": "linear"

  13192.                 },

  13193.                 "showPoints": "never",

  13194.                 "spanNulls": false,

  13195.                 "stacking": {

  13196.                   "group": "A",

  13197.                   "mode": "none"

  13198.                 },

  13199.                 "thresholdsStyle": {

  13200.                   "mode": "off"

  13201.                 }

  13202.               },

  13203.               "mappings": [],

  13204.               "thresholds": {

  13205.                 "mode": "absolute",

  13206.                 "steps": [

  13207.                   {

  13208.                     "color": "green"

  13209.                   }

  13210.                 ]

  13211.               },

  13212.               "unit": "hertz"

  13213.             },

  13214.             "overrides": []

  13215.           },

  13216.           "gridPos": {

  13217.             "h": 8,

  13218.             "w": 12,

  13219.             "x": 12,

  13220.             "y": 55

  13221.           },

  13222.           "id": 223,

  13223.           "options": {

  13224.             "legend": {

  13225.               "calcs": [],

  13226.               "displayMode": "list",

  13227.               "placement": "bottom",

  13228.               "showLegend": true

  13229.             },

  13230.             "tooltip": {

  13231.               "mode": "multi",

  13232.               "sort": "desc"

  13233.             }

  13234.           },

  13235.           "pluginVersion": "9.0.4",

  13236.           "targets": [

  13237.             {

  13238.               "datasource": {

  13239.                 "uid": "${DS_PROMETHEUS}"

  13240.               },

  13241.               "editorMode": "code",

  13242.               "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])",

  13243.               "interval": "",

  13244.               "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",

  13245.               "range": true,

  13246.               "refId": "A"

  13247.             }

  13248.           ],

  13249.           "title": "External Cache Miss Rate",

  13250.           "type": "timeseries"

  13251.         }

  13252.       ],

  13253.       "targets": [

  13254.         {

  13255.           "datasource": {

  13256.             "type": "prometheus",

  13257.             "uid": "${DS_PROMETHEUS}"

  13258.           },

  13259.           "refId": "A"

  13260.         }

  13261.       ],

  13262.       "title": "External Cache",

  13263.       "type": "row"

  13264.     }

  13265.   ],

  13266.   "refresh": false,

  13267.   "schemaVersion": 37,

  13268.   "style": "dark",

  13269.   "tags": [

  13270.     "matrix"

  13271.   ],

  13272.   "templating": {

  13273.     "list": [

  13274.       {

  13275.         "current": {

  13276.           "selected": false,

  13277.           "text": "default",

  13278.           "value": "default"

  13279.         },

  13280.         "hide": 0,

  13281.         "includeAll": false,

  13282.         "multi": false,

  13283.         "name": "DS_PROMETHEUS",

  13284.         "label": "Datasource",

  13285.         "options": [],

  13286.         "query": "prometheus",

  13287.         "queryValue": "",

  13288.         "refresh": 1,

  13289.         "regex": "",

  13290.         "skipUrlSync": false,

  13291.         "type": "datasource"

  13292.       },

  13293.       {

  13294.         "allFormat": "glob",

  13295.         "auto": true,

  13296.         "auto_count": 100,

  13297.         "auto_min": "30s",

  13298.         "current": {

  13299.           "selected": false,

  13300.           "text": "auto",

  13301.           "value": "$__auto_interval_bucket_size"

  13302.         },

  13303.         "hide": 0,

  13304.         "includeAll": false,

  13305.         "label": "Bucket Size",

  13306.         "multi": false,

  13307.         "multiFormat": "glob",

  13308.         "name": "bucket_size",

  13309.         "options": [

  13310.           {

  13311.             "selected": true,

  13312.             "text": "auto",

  13313.             "value": "$__auto_interval_bucket_size"

  13314.           },

  13315.           {

  13316.             "selected": false,

  13317.             "text": "30s",

  13318.             "value": "30s"

  13319.           },

  13320.           {

  13321.             "selected": false,

  13322.             "text": "1m",

  13323.             "value": "1m"

  13324.           },

  13325.           {

  13326.             "selected": false,

  13327.             "text": "2m",

  13328.             "value": "2m"

  13329.           },

  13330.           {

  13331.             "selected": false,

  13332.             "text": "5m",

  13333.             "value": "5m"

  13334.           },

  13335.           {

  13336.             "selected": false,

  13337.             "text": "10m",

  13338.             "value": "10m"

  13339.           },

  13340.           {

  13341.             "selected": false,

  13342.             "text": "15m",

  13343.             "value": "15m"

  13344.           }

  13345.         ],

  13346.         "query": "30s,1m,2m,5m,10m,15m",

  13347.         "queryValue": "",

  13348.         "refresh": 2,

  13349.         "skipUrlSync": false,

  13350.         "type": "interval"

  13351.       },

  13352.       {

  13353.         "current": {},

  13354.         "datasource": {

  13355.           "uid": "${DS_PROMETHEUS}"

  13356.         },

  13357.         "definition": "",

  13358.         "hide": 0,

  13359.         "includeAll": false,

  13360.         "multi": false,

  13361.         "name": "instance",

  13362.         "options": [],

  13363.         "query": {

  13364.           "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)",

  13365.           "refId": "Prometheus-instance-Variable-Query"

  13366.         },

  13367.         "refresh": 2,

  13368.         "regex": "",

  13369.         "skipUrlSync": false,

  13370.         "sort": 1,

  13371.         "tagValuesQuery": "",

  13372.         "tagsQuery": "",

  13373.         "type": "query",

  13374.         "useTags": false

  13375.       },

  13376.       {

  13377.         "allFormat": "regex wildcard",

  13378.         "allValue": "",

  13379.         "current": {},

  13380.         "datasource": {

  13381.           "uid": "${DS_PROMETHEUS}"

  13382.         },

  13383.         "definition": "",

  13384.         "hide": 0,

  13385.         "hideLabel": false,

  13386.         "includeAll": true,

  13387.         "label": "Job",

  13388.         "multi": true,

  13389.         "multiFormat": "regex values",

  13390.         "name": "job",

  13391.         "options": [],

  13392.         "query": {

  13393.           "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)",

  13394.           "refId": "Prometheus-job-Variable-Query"

  13395.         },

  13396.         "refresh": 2,

  13397.         "refresh_on_load": false,

  13398.         "regex": "",

  13399.         "skipUrlSync": false,

  13400.         "sort": 1,

  13401.         "tagValuesQuery": "",

  13402.         "tagsQuery": "",

  13403.         "type": "query",

  13404.         "useTags": false

  13405.       },

  13406.       {

  13407.         "allFormat": "regex wildcard",

  13408.         "allValue": ".*",

  13409.         "current": {},

  13410.         "datasource": {

  13411.           "uid": "${DS_PROMETHEUS}"

  13412.         },

  13413.         "definition": "",

  13414.         "hide": 0,

  13415.         "hideLabel": false,

  13416.         "includeAll": true,

  13417.         "label": "",

  13418.         "multi": true,

  13419.         "multiFormat": "regex values",

  13420.         "name": "index",

  13421.         "options": [],

  13422.         "query": {

  13423.           "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)",

  13424.           "refId": "Prometheus-index-Variable-Query"

  13425.         },

  13426.         "refresh": 2,

  13427.         "refresh_on_load": false,

  13428.         "regex": "",

  13429.         "skipUrlSync": false,

  13430.         "sort": 3,

  13431.         "tagValuesQuery": "",

  13432.         "tagsQuery": "",

  13433.         "type": "query",

  13434.         "useTags": false

  13435.       }

  13436.     ]

  13437.   },

  13438.   "time": {

  13439.     "from": "now-3h",

  13440.     "to": "now"

  13441.   },

  13442.   "timepicker": {

  13443.     "now": true,

  13444.     "refresh_intervals": [

  13445.       "5s",

  13446.       "10s",

  13447.       "30s",

  13448.       "1m",

  13449.       "5m",

  13450.       "15m",

  13451.       "30m",

  13452.       "1h",

  13453.       "2h",

  13454.       "1d"

  13455.     ],

  13456.     "time_options": [

  13457.       "5m",

  13458.       "15m",

  13459.       "1h",

  13460.       "6h",

  13461.       "12h",

  13462.       "24h",

  13463.       "2d",

  13464.       "7d",

  13465.       "30d"

  13466.     ]

  13467.   },

  13468.   "timezone": "",

  13469.   "title": "Synapse",

  13470.   "uid": "000000012",

  13471.   "version": 160,

  13472.   "weekStart": ""

  13473. }