From ee70e99ea1a547f4ce4eaeea99feba129bd9d271 Mon Sep 17 00:00:00 2001 From: 0x1d Date: Mon, 27 Oct 2025 21:38:20 +0100 Subject: [PATCH] feat: multiarch build --- Makefile | 34 +++++- README.md | 24 ++++ docs/DOCKER.md | 180 ++++++++++++++++++++++++++++ docs/MULTIARCH.md | 235 +++++++++++++++++++++++++++++++++++++ scripts/setup-multiarch.sh | 64 ++++++++++ 5 files changed, 535 insertions(+), 2 deletions(-) create mode 100644 docs/DOCKER.md create mode 100644 docs/MULTIARCH.md create mode 100755 scripts/setup-multiarch.sh diff --git a/Makefile b/Makefile index e4a97fe..3a5f525 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build push up down logs ps restart clean nomad-start nomad-stop nomad-status nomad-ui nomad-job-run nomad-job-stop nomad-logs +.PHONY: help build push up down logs ps restart clean nomad-start nomad-stop nomad-status nomad-ui nomad-job-run nomad-job-stop nomad-logs build-multiarch push-multiarch # Default target help: @@ -7,8 +7,10 @@ help: @echo "Targets:" @echo " make build - Build all Docker images" @echo " make push - Build and push all Docker images to Docker Hub" + @echo " make build-multiarch - Build multiarch images (amd64 & arm64)" + @echo " make push-multiarch - Build and push multiarch images (amd64 & arm64)" @echo " make up - Start all services" - @echo " make down - Stop all services" + @echo " make down - Stop all services" @echo " make logs - View logs from all services" @echo " make ps - Show service status" @echo " make restart - Restart all services" @@ -45,6 +47,34 @@ push: @cd ../spore-ui && make docker-build && make docker-push @echo "All images pushed successfully!" +# Build all images with multiarch support +build-multiarch: IMAGE_TAG ?= latest +build-multiarch: + @echo "Building multiarch images for amd64 and arm64..." + @echo "Building spore-gateway..." + @cd ../spore-gateway && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "Building spore-registry..." + @cd ../spore-registry && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "Building spore-ledlab..." + @cd ../spore-ledlab && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "Building spore-ui..." + @cd ../spore-ui && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "All multiarch images built successfully!" + +# Build and push all images with multiarch support +push-multiarch: IMAGE_TAG ?= latest +push-multiarch: + @echo "Building and pushing multiarch images to Docker Hub..." + @echo "Building and pushing spore-gateway..." + @cd ../spore-gateway && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "Building and pushing spore-registry..." + @cd ../spore-registry && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "Building and pushing spore-ledlab..." + @cd ../spore-ledlab && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "Building and pushing spore-ui..." + @cd ../spore-ui && make docker-build-multiarch IMAGE_TAG=$(IMAGE_TAG) + @echo "All multiarch images pushed successfully!" + # Start all services up: docker compose up -d diff --git a/README.md b/README.md index 81d15b6..d5ddb6f 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,30 @@ cd ../spore-ui && make docker-build make push ``` +### Building Multiarch Images (amd64 & arm64) + +For Raspberry Pi and other ARM devices, build multiarch images: + +```bash +# One-time setup (run the setup script) +./scripts/setup-multiarch.sh + +# Or setup manually: +docker run --rm --privileged tonistiigi/binfmt:latest --install all +docker buildx create --name multiarch --use +docker buildx inspect --bootstrap + +# Build and push all multiarch images +cd spore-deployment +make push-multiarch + +# Or build without pushing +make build-multiarch + +# Or build for specific tag +make push-multiarch IMAGE_TAG=v1.0.0 +``` + ### Environment Variables #### Docker Compose diff --git a/docs/DOCKER.md b/docs/DOCKER.md new file mode 100644 index 0000000..696458d --- /dev/null +++ b/docs/DOCKER.md @@ -0,0 +1,180 @@ +# SPORE Docker Build Guide + +This document provides instructions for building and running SPORE services as Docker containers. + +## Prerequisites + +- Docker installed on your system +- Docker Compose (optional, for running multiple services) + +## Building Docker Images + +### Using Makefiles + +Each project includes a Makefile with Docker targets for easy building and pushing: + +```bash +# Build Docker image (single architecture) +make docker-build + +# Push Docker image (optional, requires DOCKER_REGISTRY variable) +make docker-push +``` + +**Optional**: Specify a Docker registry and tag: + +```bash +# Build with custom tag +make docker-build IMAGE_TAG=v1.0.0 + +# Push to custom registry +make docker-push DOCKER_REGISTRY=your-registry.com IMAGE_TAG=v1.0.0 +``` + +### Building Multiarch Images (amd64 & arm64) + +For Raspberry Pi and other ARM devices, build multiarch images: + +```bash +# One-time setup for buildx +docker run --rm --privileged tonistiigi/binfmt:latest --install all +docker buildx create --name multiarch --use +docker buildx inspect --bootstrap + +# Build and push multiarch images +make docker-build-multiarch IMAGE_TAG=latest +``` + +This builds images for both `linux/amd64` and `linux/arm64` architectures simultaneously. + +### Direct Docker Commands + +### spore-gateway + +From the `spore-gateway` directory: + +```bash +docker build -t spore-gateway:latest . +docker run -p 3001:3001 -p 4210:4210/udp spore-gateway:latest +``` + +### spore-ledlab + +From the `spore-ledlab` directory: + +```bash +docker build -t spore-ledlab:latest . +docker run -p 8080:8080 --env GATEWAY_URL=http://localhost:3001 spore-ledlab:latest +``` + +### spore-registry + +From the `spore-registry` directory: + +```bash +docker build -t spore-registry:latest . +docker run -p 8080:8080 -v registry-data:/data/registry spore-registry:latest +``` + +Note: The registry data is stored in a Docker volume for persistence. + +### spore-ui + +From the `spore-ui` directory: + +```bash +docker build -t spore-ui:latest . +docker run -p 3000:3000 spore-ui:latest +``` + +## Environment Variables + +### spore-ledlab + +- `PORT`: Server port (default: 8080) +- `UDP_PORT`: UDP discovery port (default: 4210) +- `GATEWAY_URL`: Gateway URL (default: http://localhost:3001) +- `FILTER_APP_LABEL`: Filter app label (default: pixelstream) +- `MATRIX_WIDTH`: Matrix width (default: 16) +- `MATRIX_HEIGHT`: Matrix height (default: 16) + +### spore-gateway + +- HTTP server port: 3001 (override with `-port` flag) +- UDP discovery port: 4210 (override with `-udp-port` flag) +- Log level (default: info, override with `-log-level` flag) + +### spore-ui + +- `PORT`: Server port (default: 3000) + +## Docker Compose (Optional) + +You can create a `docker-compose.yml` file to run all services together: + +```yaml +version: '3.8' + +services: + gateway: + build: ./spore-gateway + ports: + - "3001:3001" + - "4210:4210/udp" + + ledlab: + build: ./spore-ledlab + ports: + - "8080:8080" + environment: + - GATEWAY_URL=http://gateway:3001 + depends_on: + - gateway + + registry: + build: ./spore-registry + ports: + - "8090:8080" + volumes: + - registry-data:/data/registry + + ui: + build: ./spore-ui + ports: + - "3000:3000" + +volumes: + registry-data: +``` + +Then run: + +```bash +docker-compose up -d +``` + +## Troubleshooting + +### Port Conflicts + +If ports are already in use, modify the port mappings in the `docker run` commands or in the `docker-compose.yml` file. + +### Registry Data Persistence + +The spore-registry service uses a Docker volume to persist data. To backup or restore: + +```bash +# Backup +docker run --rm -v registry-data:/data -v $(pwd):/backup alpine tar czf /backup/registry-backup.tar.gz /data + +# Restore +docker run --rm -v registry-data:/data -v $(pwd):/backup alpine tar xzf /backup/registry-backup.tar.gz -C / +``` + +### Network Configuration + +When running services in separate containers, make sure they can reach each other. Either: +- Use Docker networking with docker-compose +- Use host networking: `--network host` +- Update service URLs to point to appropriate hosts/containers + diff --git a/docs/MULTIARCH.md b/docs/MULTIARCH.md new file mode 100644 index 0000000..b4f5df5 --- /dev/null +++ b/docs/MULTIARCH.md @@ -0,0 +1,235 @@ +# Multiarch Docker Images for SPORE + +This guide explains how to build and deploy SPORE Docker images that support multiple architectures (amd64 and arm64). + +## Overview + +SPORE services can run on both x86_64 (amd64) and ARM64 systems (including Raspberry Pi). To support both architectures, we use Docker's buildx feature to create multiarch images. + +## Supported Architectures + +- **linux/amd64** - Intel/AMD 64-bit processors +- **linux/arm64** - ARM 64-bit processors (Raspberry Pi 4+, Apple Silicon, etc.) + +## Setup (One-Time) + +### Option 1: Automated Setup Script + +```bash +cd spore-deployment +./scripts/setup-multiarch.sh +``` + +### Option 2: Manual Setup + +```bash +# Install QEMU emulation for cross-architecture builds +docker run --rm --privileged tonistiigi/binfmt:latest --install all + +# Create and configure the multiarch builder +docker buildx create --name multiarch --use +docker buildx inspect --bootstrap + +# Verify setup +docker buildx ls +``` + +You should see a builder named `multiarch` with support for `linux/amd64` and `linux/arm64`. + +## Building Multiarch Images + +### All Services at Once + +From the `spore-deployment` directory: + +```bash +# Build and push all multiarch images +make push-multiarch + +# Build without pushing (for local testing) +make build-multiarch + +# Build with custom tag +make push-multiarch IMAGE_TAG=v1.0.0 +``` + +### Individual Services + +From each service directory: + +```bash +# spore-gateway +cd spore-gateway +make docker-build-multiarch + +# spore-registry +cd spore-registry +make docker-build-multiarch + +# spore-ledlab +cd spore-ledlab +make docker-build-multiarch + +# spore-ui +cd spore-ui +make docker-build-multiarch +``` + +## How It Works + +1. **Docker Buildx** - Uses Docker's extended build capabilities +2. **QEMU Emulation** - Allows building ARM images on x86_64 machines +3. **Multi-platform build** - Creates a single image manifest supporting multiple architectures +4. **Automatic selection** - Docker automatically pulls the correct architecture for the host system + +## Verifying Multiarch Images + +Check which architectures are supported: + +```bash +docker buildx imagetools inspect wirelos/spore-gateway:latest +``` + +Output should show both: +- linux/amd64 +- linux/arm64 + +## Using Multiarch Images + +Once pushed to a registry, images are automatically used by Docker based on the host architecture: + +**On x86_64 systems:** +```bash +docker pull wirelos/spore-gateway:latest +# Docker automatically uses amd64 version +``` + +**On Raspberry Pi (ARM64):** +```bash +docker pull wirelos/spore-gateway:latest +# Docker automatically uses arm64 version +``` + +## Docker Compose + +The existing `docker-compose.yml` works automatically with multiarch images: + +```bash +cd spore-deployment +make up +``` + +Docker Compose will pull the correct architecture for your system. + +## Nomad Deployment + +Multiarch images work automatically with Nomad: + +```bash +cd spore-deployment +make nomad-job-run +``` + +Nomad will schedule pods based on available nodes and their architectures. + +## Troubleshooting + +### Check Available Platforms + +```bash +docker buildx inspect multiarch +``` + +### Switch Builders + +```bash +# Use multiarch builder +docker buildx use multiarch + +# Use default builder +docker buildx use default +``` + +### Remove and Recreate Builder + +```bash +docker buildx rm multiarch +docker buildx create --name multiarch --use +docker buildx inspect --bootstrap +``` + +### Build for Specific Platform Only + +```bash +# Build only for ARM64 +docker buildx build --platform linux/arm64 -t wirelos/spore-gateway:latest . + +# Build only for AMD64 +docker buildx build --platform linux/amd64 -t wirelos/spore-gateway:latest . +``` + +### Common Issues + +#### Error: "multiple platforms feature is currently not supported" +- Solution: Enable buildkit: `export DOCKER_CLI_EXPERIMENTAL=enabled` + +#### Error: "failed to solve: failed to compute cache key" +- Solution: Make sure all source files are present in build context + +#### Slow builds on first run +- Normal: QEMU emulation is slower than native builds +- Subsequent builds use Docker layer cache and are faster + +## CI/CD Integration + +### GitHub Actions Example + +```yaml +name: Build and Push Multiarch + +on: + push: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build and push + run: | + cd spore-deployment + make push-multiarch IMAGE_TAG=${{ github.sha }} +``` + +## Security Considerations + +- Multiarch images are signed for each platform +- When using third-party base images, ensure they also support multiarch +- Test on both architectures before production deployment + +## Performance Notes + +### Build Performance + +- Native builds (build on the target architecture): Fastest +- Cross-compilation (build on different architecture): Slower but convenient +- Emulation via QEMU: Slowest but allows building all architectures on one machine + +### Runtime Performance + +- No difference: Once the correct architecture image is pulled, performance is the same as single-arch builds + +## Additional Resources + +- [Docker Buildx Documentation](https://docs.docker.com/buildx/) +- [Docker Multiarch Tutorial](https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/) +- [QEMU Documentation](https://www.qemu.org/documentation/) + diff --git a/scripts/setup-multiarch.sh b/scripts/setup-multiarch.sh new file mode 100755 index 0000000..11b3e7b --- /dev/null +++ b/scripts/setup-multiarch.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# SPORE Multiarch Build Setup Script +# This script sets up Docker buildx for multiarch image building + +set -e + +echo "🚀 SPORE Multiarch Build Setup" +echo "================================" +echo "" + +# Check if buildx is installed +if ! command -v docker &> /dev/null; then + echo "❌ Docker is not installed. Please install Docker first." + exit 1 +fi + +echo "✅ Docker is installed" + +# Install QEMU emulation for cross-architecture builds +echo "" +echo "📦 Installing QEMU emulation..." +docker run --rm --privileged tonistiigi/binfmt:latest --install all + +echo "✅ QEMU emulation installed" + +# Check if multiarch builder already exists +EXISTING_BUILDER=$(docker buildx ls --format '{{.Name}}' | grep -w multiarch || true) + +if [ -z "$EXISTING_BUILDER" ]; then + echo "" + echo "📦 Creating multiarch builder..." + docker buildx create --name multiarch --use + echo "✅ Multiarch builder created" +else + echo "" + echo "✅ Multiarch builder already exists" + echo "📦 Switching to multiarch builder..." + docker buildx use multiarch +fi + +echo "" +echo "🔧 Inspecting builder..." +docker buildx inspect --bootstrap + +echo "" +echo "✅ Multiarch build setup complete!" +echo "" +echo "Available platforms:" +docker buildx inspect --bootstrap | grep "Platform:" + +echo "" +echo "📚 Usage:" +echo " Build and push all multiarch images:" +echo " cd spore-deployment" +echo " make push-multiarch" +echo "" +echo " Build without pushing:" +echo " make build-multiarch" +echo "" +echo " Build with custom tag:" +echo " make push-multiarch IMAGE_TAG=v1.0.0" +echo "" +