From fda9701e35a9fb67968d6e6e5f120555b75faed2 Mon Sep 17 00:00:00 2001 From: Patrick Balsiger Date: Tue, 23 Jan 2024 12:11:49 +0100 Subject: [PATCH] Various changes --- config.yaml | 20 +-- ctl.sh | 2 +- docker-compose.yaml | 15 +++ grafana/dashboards/drift-keeper.json | 193 ++++++++++++++++++++++++++- main.tf | 11 +- prometheus/prometheus.yml | 3 + prometheus/web.yml | 2 + 7 files changed, 226 insertions(+), 20 deletions(-) diff --git a/config.yaml b/config.yaml index 096755e..8df4a51 100644 --- a/config.yaml +++ b/config.yaml @@ -36,7 +36,7 @@ global: eventSubscriberPollingInterval: 5000 bulkAccountLoaderPollingInterval: 5000 - useJito: true + useJito: false jitoBlockEngineUrl: jitoAuthPrivateKey: @@ -58,6 +58,7 @@ enabledBots: # Trigger bot (triggers trigger orders) #- trigger + # Liquidator bot, liquidates unhealthy positions by taking over the risk (caution, you should manage risk here) # - liquidator @@ -77,12 +78,6 @@ enabledBots: # below are bot configs botConfigs: - floatingMaker: - botId: "floatingMaker" - dryRun: false - fillerPollingInterval: 6000 - metricsPort: 9464 - revertOnFailure: true fillerLite: botId: "fillerLite" @@ -90,6 +85,7 @@ botConfigs: fillerPollingInterval: 6000 metricsPort: 9464 revertOnFailure: true + simulateTxForCUEstimate: true filler: botId: "filler" @@ -102,16 +98,22 @@ botConfigs: # Default is true revertOnFailure: true + # calls simulateTransaction before sending to get an accurate CU estimate + # as well as stop before sending the transaction (Default is true) + simulateTxForCUEstimate: true + spotFiller: botId: "spot-filler" dryRun: false fillerPollingInterval: 6000 - metricsPort: 9466 + metricsPort: 9465 + revertOnFailure: true + simulateTxForCUEstimate: true liquidator: botId: "liquidator" dryRun: false - metricsPort: 9465 + metricsPort: 9466 # if true will NOT attempt to sell off any inherited positions disableAutoDerisking: false # if true will swap spot assets on jupiter if the price is better diff --git a/ctl.sh b/ctl.sh index 6c49fea..8c11f68 100755 --- a/ctl.sh +++ b/ctl.sh @@ -24,7 +24,7 @@ function droplet { terraform apply } function connect { - ssh keeper@$(terraform output -raw droplet_ip) + ssh root@$(terraform output -raw droplet_ip) } ${@:-} } diff --git a/docker-compose.yaml b/docker-compose.yaml index a42687d..d884518 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,6 +27,21 @@ services: volumes: - ./prometheus:/etc/prometheus - prom_data:/prometheus + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + expose: + - 9100 grafana: image: grafana/grafana container_name: grafana diff --git a/grafana/dashboards/drift-keeper.json b/grafana/dashboards/drift-keeper.json index 7828302..ed3a0c0 100644 --- a/grafana/dashboards/drift-keeper.json +++ b/grafana/dashboards/drift-keeper.json @@ -608,7 +608,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -646,6 +646,110 @@ "x": 0, "y": 23 }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "successful_fills_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Successful Fills Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.instance}", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 23 + }, "id": 1, "options": { "legend": { @@ -726,8 +830,8 @@ "gridPos": { "h": 12, "w": 12, - "x": 12, - "y": 23 + "x": 0, + "y": 35 }, "id": 5, "options": { @@ -762,6 +866,85 @@ ], "title": "Attempted Fill Duration", "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "custom": { + "fillOpacity": 50, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "displayName": "${__field.labels.instance}", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 14, + "options": { + "bucketOffset": 0, + "combine": false, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(sdk_call_duration_histogram_sum[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "SDK Call Duration", + "type": "histogram" } ], "refresh": "", @@ -771,13 +954,13 @@ "list": [] }, "time": { - "from": "now-24h", + "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Drift Keeper", "uid": "f4c3a630-ffd0-41c8-a36a-52e0771b77fb", - "version": 5, + "version": 3, "weekStart": "" } \ No newline at end of file diff --git a/main.tf b/main.tf index ade5027..e226c30 100644 --- a/main.tf +++ b/main.tf @@ -10,8 +10,9 @@ terraform { variable "do_token" {} variable "config" { default = { - ntp_server = "ntp.amsterdam.jito.wtf" - docker_image = "wirelos/drift-keeper:mainnet-beta" + region = "ams3" + ntp_server = "ntp.amsterdam.jito.wtf" + docker_image = "wirelos/drift-keeper:mainnet-beta" } } @@ -35,12 +36,12 @@ resource "digitalocean_ssh_key" "default" { resource "digitalocean_droplet" "keeper" { image = "ubuntu-23-10-x64" name = "drift-keeper" - region = "ams3" + region = var.config.region size = "s-1vcpu-1gb-intel" ssh_keys = [digitalocean_ssh_key.default.fingerprint] user_data = local.user_data } output "droplet_ip" { - value = digitalocean_droplet.keeper.ipv4_address -} \ No newline at end of file + value = digitalocean_droplet.keeper.ipv4_address +} diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 47b8025..589cabf 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -10,6 +10,9 @@ alerting: timeout: 10s api_version: v1 scrape_configs: +- job_name: node + static_configs: + - targets: ['node-exporter:9100'] - job_name: keeper honor_timestamps: true scrape_interval: 15s diff --git a/prometheus/web.yml b/prometheus/web.yml index 4d3b9b2..91b08ed 100644 --- a/prometheus/web.yml +++ b/prometheus/web.yml @@ -1,2 +1,4 @@ basic_auth_users: + # bcrypt "prompass" + # e.g. htpasswd -bnBC 10 "" prompass | tr -d ':\n' prom: $2y$10$oWtHm79bh0D1CnNC4brGiOU7y6MbYa6cgklF/g6ek9YZYkgXfeOIu \ No newline at end of file