From 84735d67dd3731f2feae46bec237e0e86998429c Mon Sep 17 00:00:00 2001 From: 0x1d Date: Mon, 27 Oct 2025 15:06:57 +0100 Subject: [PATCH] feat: Compose and Nomad deployment --- .gitignore | 7 + Makefile | 123 ++++++++++++ README.md | 383 +++++++++++++++++++++++++++++++++++++ config/mqtt/mosquitto.conf | 10 + config/nomad/nomad.hcl | 60 ++++++ docker-compose.yml | 105 ++++++++++ nomad/README.md | 234 ++++++++++++++++++++++ nomad/spore.hcl | 145 ++++++++++++++ 8 files changed, 1067 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 config/mqtt/mosquitto.conf create mode 100644 config/nomad/nomad.hcl create mode 100644 docker-compose.yml create mode 100644 nomad/README.md create mode 100644 nomad/spore.hcl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..27c4c31 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.env +*.log +docker-compose.override.yml +data/ +nomad-data/ +nomad.log +alloc_mounts/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2194892 --- /dev/null +++ b/Makefile @@ -0,0 +1,123 @@ +.PHONY: help build push up down logs ps restart clean nomad-start nomad-stop nomad-status nomad-ui nomad-job-run nomad-job-stop nomad-logs + +# Default target +help: + @echo "SPORE Deployment Makefile" + @echo "" + @echo "Targets:" + @echo " make build - Build all Docker images" + @echo " make push - Build and push all Docker images to Docker Hub" + @echo " make up - Start all services" + @echo " make down - Stop all services" + @echo " make logs - View logs from all services" + @echo " make ps - Show service status" + @echo " make restart - Restart all services" + @echo " make clean - Stop services and remove data directory" + @echo " make gateway-logs - View gateway logs" + @echo " make ledlab-logs - View LEDLab logs" + @echo " make registry-logs - View registry logs" + @echo " make ui-logs - View UI logs" + @echo " make mqtt-logs - View MQTT broker logs" + @echo "" + @echo "Nomad targets:" + @echo " make nomad-start - Start Nomad locally" + @echo " make nomad-stop - Stop Nomad" + @echo " make nomad-status - Check Nomad status" + @echo " make nomad-ui - Open Nomad UI in browser" + @echo " make nomad-job-run - Deploy SPORE job to Nomad" + @echo " make nomad-job-stop - Stop SPORE job in Nomad" + @echo " make nomad-logs - View Nomad job logs" + +# Build all images +build: + docker compose build + +# Build and push all images to Docker Hub +push: + @echo "Building and pushing all SPORE images to Docker Hub..." + @echo "Building spore-gateway..." + @cd ../spore-gateway && make docker-build && make docker-push + @echo "Building spore-registry..." + @cd ../spore-registry && make docker-build && make docker-push + @echo "Building spore-ledlab..." + @cd ../spore-ledlab && make docker-build && make docker-push + @echo "Building spore-ui..." + @cd ../spore-ui && make docker-build && make docker-push + @echo "All images pushed successfully!" + +# Start all services +up: + docker compose up -d + +# Stop all services +down: + docker compose down + +# View logs +logs: + docker compose logs -f + +# Show service status +ps: + docker compose ps + +# Restart all services +restart: + docker compose restart + +# Clean (stop and remove data) +clean: + @echo "Stopping services..." + @docker compose down + @echo "Removing data directory..." + @rm -rf ./data/ + @echo "Clean complete" + +# Service-specific logs +gateway-logs: + docker compose logs -f gateway + +ledlab-logs: + docker compose logs -f ledlab + +registry-logs: + docker compose logs -f registry + +ui-logs: + docker compose logs -f ui + +mqtt-logs: + docker compose logs -f mqtt + +# Nomad targets +nomad-start: + @echo "Starting Nomad..." + @mkdir -p nomad-data + @sudo nomad agent -dev -config=./config/nomad/nomad.hcl -data-dir=$(PWD)/nomad-data + @sleep 2 + @echo "Nomad started at http://localhost:4646" + +nomad-stop: + @echo "Stopping Nomad..." + @pkill -f "nomad agent" || true + @echo "Stopped" + +nomad-status: + @nomad status || echo "Nomad not running" + +nomad-ui: + @echo "Opening Nomad UI at http://localhost:4646" + @xdg-open http://localhost:4646 2>/dev/null || open http://localhost:4646 2>/dev/null || echo "Please open http://localhost:4646 in your browser" + +nomad-job-run: + @echo "Deploying SPORE job to Nomad..." + @nomad job run ./nomad/spore.hcl + +nomad-job-stop: + @echo "Stopping SPORE job in Nomad..." + @nomad job stop spore + +nomad-logs: + @echo "Showing logs for SPORE jobs..." + @nomad job logs spore + diff --git a/README.md b/README.md new file mode 100644 index 0000000..81d15b6 --- /dev/null +++ b/README.md @@ -0,0 +1,383 @@ +# SPORE Deployment + +Complete deployment configurations for running the SPORE stack with Docker Compose or HashiCorp Nomad. + +## Overview + +This repository provides two deployment options: + +1. **Docker Compose** - Simple deployment with docker-compose for development and production +2. **Nomad** - Production-ready orchestration using HashiCorp Nomad + +Both deployments include all SPORE services with proper networking, volumes, and service discovery. + +## Services + +- **mosquitto**: MQTT broker for message routing (port 1883, 9001) +- **spore-gateway**: Node discovery and WebSocket gateway with MQTT integration (port 3001, UDP 4210) +- **spore-registry**: Firmware registry service (port 8090) +- **spore-ledlab**: LED animation studio (port 8080) +- **spore-ui**: Web UI for cluster management (port 3000) + +## Prerequisites + +### For Docker Compose +- Docker and Docker Compose installed +- Available ports: 1883, 3000, 3001, 4210, 8080, 8090 + +### For Nomad +- HashiCorp Nomad installed +- Docker installed (Nomad uses Docker to run containers) +- Available ports: 1883, 3000, 3001, 4210, 8080, 8090, 4646 + +## Deployment Options + +### Option 1: Docker Compose (Recommended for development) + +Simple, single-node deployment with docker-compose. + +#### Quick Start + +```bash +cd spore-deployment + +# Start all services +docker compose up -d + +# View logs +docker compose logs -f + +# Stop all services +docker compose down +``` + +#### Services Access + +- **Web UI**: http://localhost:3000 +- **LED Lab**: http://localhost:8080 +- **Registry API**: http://localhost:8090 +- **Gateway API**: http://localhost:3001 +- **MQTT Broker**: tcp://localhost:1883 (WebSocket: ws://localhost:9001) + +#### Data Storage + +All data is persisted in local `./data/` directory: +- Registry data: `./data/registry` +- MQTT data: `./data/mqtt/` + +#### Management + +```bash +# View logs for all services +make logs + +# View logs for specific service +make gateway-logs +make registry-logs +make ledlab-logs +make ui-logs +make mqtt-logs + +# Restart services +make restart + +# Clean (stop and remove all data) +make clean +``` + +### Option 2: Nomad (Recommended for production) + +Distributed deployment with HashiCorp Nomad. + +#### Quick Start + +```bash +cd spore-deployment + +# Start Nomad in dev mode +make nomad-start + +# Build Docker images first +cd ../spore-gateway && make docker-build +cd ../spore-ledlab && make docker-build +cd ../spore-registry && make docker-build +cd ../spore-ui && make docker-build + +# Deploy the job +make nomad-job-run + +# View status +make nomad-status +``` + +#### Services Access + +Same ports as Docker Compose deployment. + +#### Nomad UI + +Access the Nomad UI at http://localhost:4646 to manage and monitor jobs. + +#### Management + +```bash +# View Nomad UI +make nomad-ui + +# View job logs +make nomad-logs + +# Stop the job +make nomad-job-stop + +# Restart Nomad +make nomad-stop +make nomad-start +``` + +#### Nomad Configuration + +Nomad configuration is in `config/nomad/nomad.hcl`: +- Dev mode with single agent +- Docker driver enabled +- Host networking for all services +- UI enabled on port 4646 + +See [nomad/README.md](./nomad/README.md) for detailed Nomad documentation. + +## Common Tasks + +### Building Images + +Both deployment methods require Docker images to be built first. + +```bash +# Using make in each project +cd ../spore-gateway && make docker-build +cd ../spore-ledlab && make docker-build +cd ../spore-registry && make docker-build +cd ../spore-ui && make docker-build + +# Or build and push all at once from deployment directory +make push +``` + +### Environment Variables + +#### Docker Compose + +Create a `.env` file to override defaults: + +```env +LOG_LEVEL=debug +MATRIX_WIDTH=32 +MATRIX_HEIGHT=32 +FPS=30 +``` + +Then run: +```bash +docker compose --env-file .env up -d +``` + +#### Nomad + +Edit `nomad/spore.hcl` to modify environment variables or resource allocations. + +### Data Management + +#### Backup + +```bash +# Backup all data +tar czf spore-backup.tar.gz ./data/ + +# Backup registry only +tar czf registry-backup.tar.gz ./data/registry/ +``` + +#### Restore + +```bash +# Restore all data +tar xzf spore-backup.tar.gz + +# Restore registry only +tar xzf registry-backup.tar.gz +``` + +#### Reset Data + +```bash +# Docker Compose +make clean + +# Nomad +rm -rf nomad-data/ +``` + +## Logs + +### Docker Compose + +```bash +# All services +docker compose logs -f + +# Specific service +docker compose logs -f gateway +docker compose logs -f registry +``` + +### Nomad + +```bash +# All job logs +make nomad-logs + +# Specific task +nomad alloc logs + +# Follow logs +nomad alloc logs -f +``` + +## MQTT Integration + +Both deployments enable MQTT integration for the gateway service. + +### Publishing to MQTT + +```bash +# Using mosquitto_pub +mosquitto_pub -h localhost -t "spore/cluster/broadcast" -m '{"command":"update"}' + +# Send to specific node +mosquitto_pub -h localhost -t "spore/nodes/192.168.1.100/command" -m '{"action":"update"}' +``` + +### Monitoring MQTT + +```bash +# Subscribe to all SPORE topics +mosquitto_sub -h localhost -t "spore/#" -v + +# Watch specific topic +mosquitto_sub -h localhost -t "spore/cluster/status" -v +``` + +### MQTT Authentication + +Enable authentication by setting environment variables: + +**Docker Compose:** +```yaml +gateway: + environment: + - MQTT_SERVER=tcp://localhost:1883 + - MQTT_USER=username + - MQTT_PASSWORD=password +``` + +**Nomad:** +Edit `nomad/spore.hcl` to add MQTT_USER and MQTT_PASSWORD env vars. + +## Troubleshooting + +### Port Conflicts + +If ports are already in use: + +**Docker Compose:** Edit port mappings in `docker-compose.yml` + +**Nomad:** Edit ports in `nomad/spore.hcl` + +### Services Not Starting + +**Docker Compose:** +```bash +# Check logs +make logs + +# Check container status +docker compose ps +``` + +**Nomad:** +```bash +# Check job status +make nomad-status + +# Check allocation logs +nomad alloc logs + +# Check allocation status +nomad alloc status +``` + +### Network Issues + +Both deployments use host networking for proper UDP broadcast support. If services can't communicate: + +1. Check firewall settings +2. Verify ports are not blocked +3. Check service logs for connection errors + +### Images Not Found + +Ensure Docker images are built before deploying: + +```bash +# Build all images +make push + +# Or build individually +cd ../spore-gateway && make docker-build +cd ../spore-ledlab && make docker-build +cd ../spore-registry && make docker-build +cd ../spore-ui && make docker-build +``` + +## Production Deployment + +### Docker Compose + +For production, consider: +1. Using environment-specific configuration files +2. Setting up proper logging aggregation +3. Using Docker secrets for sensitive data +4. Implementing health checks +5. Setting resource limits + +### Nomad + +For production deployment: +1. Use a multi-node Nomad cluster +2. Enable ACLs for security +3. Configure TLS for all connections +4. Use external storage for volumes +5. Set up proper monitoring and alerting +6. Configure auto-scaling based on load + +## Comparing Deployment Options + +| Feature | Docker Compose | Nomad | +|---------|---------------|-------| +| Ease of setup | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | +| Scalability | ⭐⭐ | ⭐⭐⭐⭐⭐ | +| Multi-node | No | Yes | +| Service discovery | Manual | Built-in | +| Rolling updates | Manual | Automatic | +| Resource management | Basic | Advanced | +| Production-ready | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ | + +**Recommendation:** +- Use **Docker Compose** for development, testing, or single-node deployments +- Use **Nomad** for production, multi-node clusters, or when you need advanced orchestration features + +## Additional Resources + +- [SPORE Documentation](../spore/README.md) +- [Nomad Deployment Guide](./nomad/README.md) +- [Docker Compose Documentation](https://docs.docker.com/compose/) +- [Nomad Documentation](https://www.nomadproject.io/docs) diff --git a/config/mqtt/mosquitto.conf b/config/mqtt/mosquitto.conf new file mode 100644 index 0000000..d0cc309 --- /dev/null +++ b/config/mqtt/mosquitto.conf @@ -0,0 +1,10 @@ +# ----------------------------- +# Basic Mosquitto configuration +# ----------------------------- +listener 1883 +allow_anonymous true + +# (Optional) WebSocket listener if you exposed port 9001 above +# listener 9001 +# protocol websockets +# allow_anonymous true \ No newline at end of file diff --git a/config/nomad/nomad.hcl b/config/nomad/nomad.hcl new file mode 100644 index 0000000..e88365c --- /dev/null +++ b/config/nomad/nomad.hcl @@ -0,0 +1,60 @@ +# Nomad configuration for local development +# This runs a single Nomad instance (both server and client) +# No Consul required - using Nomad's built-in service discovery + +# Data directory (use absolute path) +data_dir = "nomad-data" + +# Server configuration +server { + enabled = true + bootstrap_expect = 1 +} + +# Client configuration +client { + enabled = true + + # Host network for services that need it + host_network "host" { + cidr = "127.0.0.1/32" + reserved_ports = "3000,3001,3002,4210,8080,8090,1883,9001" + } + + # Volumes are now handled via Docker volumes +} + +# Plugin configuration (for Docker driver) +plugin "docker" { + config { + endpoint = "unix:///var/run/docker.sock" + allow_privileged = true + volumes { + enabled = true + selinuxlabel = "z" + } + } +} + +# Enable UI +ui { + enabled = true +} + +# Logging +log_level = "INFO" + +# Addresses +addresses { + http = "127.0.0.1" + rpc = "127.0.0.1" + serf = "127.0.0.1" +} + +# Ports +ports { + http = 4646 + rpc = 4647 + serf = 4648 +} + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4cebef9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +services: + mqtt: + image: eclipse-mosquitto:2.0 + container_name: spore-mqtt + ports: + - "1883:1883" + - "9001:9001" + volumes: + - ./data/mqtt/data:/mosquitto/data + - ./config/mqtt/mosquitto.conf:/mosquitto/config/mosquitto.conf + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "nc -z localhost 1883 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + + gateway: + build: + context: ../spore-gateway + dockerfile: Dockerfile + image: wirelos/spore-gateway:latest + container_name: spore-gateway + network_mode: host + environment: + - LOG_LEVEL=info + - MQTT_SERVER=tcp://localhost:1883 + - MQTT_USER= + - MQTT_PASSWORD= + depends_on: + mqtt: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "nc -z localhost 3001 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + registry: + build: + context: ../spore-registry + dockerfile: Dockerfile + image: wirelos/spore-registry:latest + container_name: spore-registry + ports: + - "3002:3002" + volumes: + - ./data/registry:/data/registry + environment: + - REGISTRY_PATH=/data/registry + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3002/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + ledlab: + build: + context: ../spore-ledlab + dockerfile: Dockerfile + image: wirelos/spore-ledlab:latest + container_name: spore-ledlab + network_mode: host + environment: + - PORT=8080 + - UDP_PORT=4210 + - GATEWAY_URL=http://localhost:3001 + - FILTER_APP_LABEL=pixelstream + - MATRIX_WIDTH=16 + - MATRIX_HEIGHT=16 + - FPS=20 + depends_on: + - gateway + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/api/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + ui: + build: + context: ../spore-ui + dockerfile: Dockerfile + image: wirelos/spore-ui:latest + container_name: spore-ui + ports: + - "3000:3000" + environment: + - PORT=3000 + depends_on: + - gateway + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + diff --git a/nomad/README.md b/nomad/README.md new file mode 100644 index 0000000..8c5caaf --- /dev/null +++ b/nomad/README.md @@ -0,0 +1,234 @@ +# SPORE Nomad Deployment + +This directory contains Nomad job specifications for deploying the SPORE stack on HashiCorp Nomad. + +## Prerequisites + +1. **Install Nomad**: https://www.nomadproject.io/downloads +2. **Docker**: Nomad uses Docker to run containers + +Note: Consul is not required. This setup uses Nomad's built-in service discovery. + +## Quick Start + +### 1. Start Nomad + +```bash +make nomad-start +``` + +This starts: +- Nomad at http://localhost:4646 + +### 2. Build Docker Images + +```bash +# Build all images +cd ../spore-gateway && make docker-build +cd ../spore-ledlab && make docker-build +cd ../spore-registry && make docker-build +cd ../spore-ui && make docker-build +``` + +### 3. Deploy the Job + +```bash +make nomad-job-run +``` + +Or manually: + +```bash +nomad job run spore.hcl +``` + +### 4. Check Status + +```bash +# Check job status +make nomad-status + +# Or use Nomad UI +make nomad-ui +``` + +## Job Specification + +The `spore.hcl` file defines 5 service groups: + +1. **mqtt** - MQTT broker (Mosquitto) +2. **gateway** - Node discovery and WebSocket gateway +3. **registry** - Firmware registry service +4. **ledlab** - LED animation studio +5. **ui** - Web UI for cluster management + +### Networking + +- **MQTT, Gateway, LEDLab**: Use host networking for UDP/WebSocket +- **Registry, UI**: Use bridge networking with mapped ports + +### Resources + +Each service has appropriate CPU and memory allocations: +- MQTT: 100 CPU, 128 MB RAM +- Gateway: 200 CPU, 256 MB RAM +- Registry: 200 CPU, 256 MB RAM +- LEDLab: 300 CPU, 512 MB RAM +- UI: 100 CPU, 256 MB RAM + +## Management + +### View Logs + +```bash +make nomad-logs +``` + +Or for specific task: + +```bash +nomad alloc logs +``` + +### Stop/Start Job + +```bash +# Stop +make nomad-job-stop + +# Start +make nomad-job-run +``` + +### Scale Services + +```bash +# Scale a specific group +nomad job scale -group mqtt -count 1 spore +``` + +### Update Job + +Edit `spore.hcl` and run: + +```bash +make nomad-job-run +``` + +Nomad will perform a rolling update if the job is running. + +## Service Discovery + +Nomad uses built-in service discovery. Services are discoverable via Nomad's internal DNS: + +- `spore-gateway.service.nomad` +- `spore-mqtt.service.nomad` +- `spore-registry.service.nomad` +- `spore-ledlab.service.nomad` +- `spore-ui.service.nomad` + +## Health Checks + +All services include health checks that: +- Verify TCP connectivity +- Check HTTP endpoints for HTTP-based services +- Automatically restart unhealthy tasks + +## Volumes + +The registry service uses a host volume mount to persist firmware data: + +```hcl +volume "registry" { + type = "host" + source = "registry" + read_only = false +} +``` + +Volume data is stored in `nomad-data/` directory. + +## Stopping Everything + +```bash +make nomad-stop +``` + +This stops both Nomad and Consul and cleans up data directories. + +## Advanced Usage + +### View Job Status + +```bash +nomad job status spore +``` + +### Inspect Task + +```bash +nomad job inspect spore +``` + +### Watch Job Events + +```bash +nomad job status spore -verbose +``` + +### Connect to Task + +```bash +nomad alloc exec /bin/sh +``` + +## Troubleshooting + +### Services Not Starting + +Check logs: + +```bash +make nomad-logs +``` + +### Port Conflicts + +If ports are in use, modify the job specification in `spore.hcl`: + +```hcl +network { + mode = "bridge" + port "http" { + static = 3001 # Change this + to = 3001 + } +} +``` + +### Images Not Found + +Ensure Docker images are built and available: + +```bash +docker images | grep wirelos +``` + +If using a registry, update the image URLs in `spore.hcl`. + + +## Production Deployment + +For production environments: + +1. **Configure ACLs**: Enable authentication +2. **Use Nomad Agents**: Deploy on multiple nodes +3. **TLS**: Enable TLS for all connections +4. **Resource Limits**: Adjust based on cluster capacity +5. **Persistence**: Use external storage for volumes + +## Next Steps + +- [Nomad Documentation](https://www.nomadproject.io/docs) +- [SPORE Documentation](../README.md) + diff --git a/nomad/spore.hcl b/nomad/spore.hcl new file mode 100644 index 0000000..6dac716 --- /dev/null +++ b/nomad/spore.hcl @@ -0,0 +1,145 @@ +job "spore" { + datacenters = ["dc1"] + type = "service" + + group "mqtt" { + count = 1 + + network { + mode = "host" + } + + task "mqtt" { + driver = "docker" + + config { + image = "eclipse-mosquitto:2.0" + network_mode = "host" + } + + resources { + cpu = 100 + memory = 128 + } + } + } + + group "gateway" { + count = 1 + + network { + mode = "host" + } + + task "gateway" { + driver = "docker" + + config { + image = "wirelos/spore-gateway:latest" + network_mode = "host" + } + + env { + LOG_LEVEL = "info" + MQTT_SERVER = "tcp://localhost:1883" + MQTT_USER = "" + MQTT_PASSWORD = "" + } + + resources { + cpu = 200 + memory = 256 + } + } + } + + group "registry" { + count = 1 + + network { + mode = "host" + } + + task "registry" { + driver = "docker" + + config { + image = "wirelos/spore-registry:latest" + network_mode = "host" + + volumes = [ + "/opt/nomad/spore/data/registry:/data/registry" + ] + } + + env { + REGISTRY_PATH = "/data/registry" + DB_PATH = "/data/registry/registry.db" + } + + resources { + cpu = 200 + memory = 256 + } + } + } + + group "ledlab" { + count = 1 + + network { + mode = "host" + } + + task "ledlab" { + driver = "docker" + + config { + image = "wirelos/spore-ledlab:latest" + network_mode = "host" + } + + env { + PORT = "8080" + UDP_PORT = "4210" + GATEWAY_URL = "http://localhost:3001" + FILTER_APP_LABEL = "pixelstream" + MATRIX_WIDTH = "16" + MATRIX_HEIGHT = "16" + FPS = "20" + } + + resources { + cpu = 300 + memory = 512 + } + } + } + + group "ui" { + count = 1 + + network { + mode = "host" + } + + task "ui" { + driver = "docker" + + config { + image = "wirelos/spore-ui:latest" + network_mode = "host" + } + + env { + PORT = "3000" + } + + resources { + cpu = 100 + memory = 256 + } + } + } +} +