Various changes

config.yaml

@@ -36,7 +36,7 @@ global:
   eventSubscriberPollingInterval: 5000
   bulkAccountLoaderPollingInterval: 5000
 
-  useJito: true
+  useJito: false
   jitoBlockEngineUrl:
   jitoAuthPrivateKey:
 
@@ -58,6 +58,7 @@ enabledBots:
 
   # Trigger bot (triggers trigger orders)
   #- trigger
+  
   # Liquidator bot, liquidates unhealthy positions by taking over the risk (caution, you should manage risk here)
   # - liquidator
 
@@ -77,12 +78,6 @@ enabledBots:
 
   # below are bot configs
 botConfigs:
-  floatingMaker:
-    botId: "floatingMaker"
-    dryRun: false
-    fillerPollingInterval: 6000
-    metricsPort: 9464
-    revertOnFailure: true
 
   fillerLite:
     botId: "fillerLite"
@@ -90,6 +85,7 @@ botConfigs:
     fillerPollingInterval: 6000
     metricsPort: 9464
     revertOnFailure: true
+    simulateTxForCUEstimate: true
 
   filler:
     botId: "filler"
@@ -102,16 +98,22 @@ botConfigs:
     # Default is true
     revertOnFailure: true
 
+    # calls simulateTransaction before sending to get an accurate CU estimate
+    # as well as stop before sending the transaction (Default is true)
+    simulateTxForCUEstimate: true
+
   spotFiller:
     botId: "spot-filler"
     dryRun: false
     fillerPollingInterval: 6000
-    metricsPort: 9466
+    metricsPort: 9465
+    revertOnFailure: true
+    simulateTxForCUEstimate: true
 
   liquidator:
     botId: "liquidator"
     dryRun: false
-    metricsPort: 9465
+    metricsPort: 9466
     # if true will NOT attempt to sell off any inherited positions
     disableAutoDerisking: false
     # if true will swap spot assets on jupiter if the price is better

ctl.sh

@@ -24,7 +24,7 @@ function droplet {
         terraform apply
     }
     function connect {
-        ssh keeper@$(terraform output -raw droplet_ip)
+        ssh root@$(terraform output -raw droplet_ip)
     }
     ${@:-}
 }

docker-compose.yaml

@@ -27,6 +27,21 @@ services:
     volumes:
       - ./prometheus:/etc/prometheus
       - prom_data:/prometheus
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: node-exporter
+    restart: unless-stopped
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+    expose:
+      - 9100
   grafana:
     image: grafana/grafana
     container_name: grafana

grafana/dashboards/drift-keeper.json

@@ -608,7 +608,7 @@
             "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
-            "pointSize": 5,
+            "pointSize": 1,
             "scaleDistribution": {
               "type": "linear"
             },
@@ -646,6 +646,110 @@
         "x": 0,
         "y": 23
       },
+      "id": 15,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "successful_fills_total",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Successful Fills Total",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 1,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "displayName": "${__field.labels.instance}",
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 12,
+        "y": 23
+      },
       "id": 1,
       "options": {
         "legend": {
@@ -726,8 +830,8 @@
       "gridPos": {
         "h": 12,
         "w": 12,
-        "x": 12,
+        "x": 0,
-        "y": 23
+        "y": 35
       },
       "id": 5,
       "options": {
@@ -762,6 +866,85 @@
       ],
       "title": "Attempted Fill Duration",
       "type": "histogram"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-BlPu"
+          },
+          "custom": {
+            "fillOpacity": 50,
+            "gradientMode": "hue",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 1
+          },
+          "displayName": "${__field.labels.instance}",
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          },
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 12,
+        "y": 35
+      },
+      "id": 14,
+      "options": {
+        "bucketOffset": 0,
+        "combine": false,
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rate(sdk_call_duration_histogram_sum[$__rate_interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "SDK Call Duration",
+      "type": "histogram"
     }
   ],
   "refresh": "",
@@ -771,13 +954,13 @@
     "list": []
   },
   "time": {
-    "from": "now-24h",
+    "from": "now-15m",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "Drift Keeper",
   "uid": "f4c3a630-ffd0-41c8-a36a-52e0771b77fb",
-  "version": 5,
+  "version": 3,
   "weekStart": ""
 }

main.tf

@@ -10,8 +10,9 @@ terraform {
 variable "do_token" {}
 variable "config" {
   default = {
-    ntp_server       = "ntp.amsterdam.jito.wtf"
+    region       = "ams3"
-    docker_image     = "wirelos/drift-keeper:mainnet-beta"
+    ntp_server   = "ntp.amsterdam.jito.wtf"
+    docker_image = "wirelos/drift-keeper:mainnet-beta"
   }
 }
 
@@ -35,12 +36,12 @@ resource "digitalocean_ssh_key" "default" {
 resource "digitalocean_droplet" "keeper" {
   image     = "ubuntu-23-10-x64"
   name      = "drift-keeper"
-  region    = "ams3"
+  region    = var.config.region
   size      = "s-1vcpu-1gb-intel"
   ssh_keys  = [digitalocean_ssh_key.default.fingerprint]
   user_data = local.user_data
 }
 
 output "droplet_ip" {
-    value = digitalocean_droplet.keeper.ipv4_address
+  value = digitalocean_droplet.keeper.ipv4_address
-}
+}

prometheus/prometheus.yml

@@ -10,6 +10,9 @@ alerting:
     timeout: 10s
     api_version: v1
 scrape_configs:
+- job_name: node
+  static_configs:
+  - targets: ['node-exporter:9100']
 - job_name: keeper
   honor_timestamps: true
   scrape_interval: 15s

prometheus/web.yml

@@ -1,2 +1,4 @@
 basic_auth_users:
+    # bcrypt "prompass"
+    # e.g. htpasswd -bnBC 10 "" prompass | tr -d ':\n'
     prom: $2y$10$oWtHm79bh0D1CnNC4brGiOU7y6MbYa6cgklF/g6ek9YZYkgXfeOIu