diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ff9862d..52074af3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -112,7 +112,7 @@ jobs: - name: Install dependencies run: brew list caddy &>/dev/null || brew install caddy - name: Run E2E install test - run: bash scripts/e2e-install-test.sh + run: CLI_BRANCH=koanf-yaml-config bash scripts/e2e-install-test.sh - name: Cleanup on failure if: failure() run: bash scripts/uninstall.sh || true diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d45e422c..06807042 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -89,45 +89,46 @@ root hard nofile 65536 ## Configuration -### Environment variables - -Hypeman can be configured using the following environment variables: - -| Variable | Description | Default | -| -------------------------- | -------------------------------------------------------------------------------------------- | ------------------ | -| `PORT` | HTTP server port | `8080` | -| `DATA_DIR` | Directory for storing VM images, volumes, and other data | `/var/lib/hypeman` | -| `BRIDGE_NAME` | Name of the network bridge for VM networking | `vmbr0` | -| `SUBNET_CIDR` | CIDR notation for the VM network subnet (gateway derived automatically) | `10.100.0.0/16` | -| `UPLINK_INTERFACE` | Host network interface to use for VM internet access | _(auto-detect)_ | -| `JWT_SECRET` | Secret key for JWT authentication (required for production) | _(empty)_ | -| `DNS_SERVER` | DNS server IP address for VMs | `1.1.1.1` | -| `MAX_CONCURRENT_BUILDS` | Maximum number of concurrent image builds | `1` | -| `MAX_OVERLAY_SIZE` | Maximum size for overlay filesystem | `100GB` | -| `ENV` | Deployment environment (filters telemetry, e.g. your name for dev) | `unset` | -| `OTEL_ENABLED` | Enable OpenTelemetry traces/metrics | `false` | -| `OTEL_ENDPOINT` | OTLP gRPC endpoint | `127.0.0.1:4317` | -| `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers) | hostname | -| `LOG_LEVEL` | Default log level (debug, info, warn, error) | `info` | -| `LOG_LEVEL_` | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC, CADDY) | inherits default | -| `CADDY_LISTEN_ADDRESS` | Address for Caddy ingress listeners | `0.0.0.0` | -| `CADDY_ADMIN_ADDRESS` | Address for Caddy admin API | `127.0.0.1` | -| `CADDY_ADMIN_PORT` | Port for Caddy admin API | `2019` | -| `CADDY_STOP_ON_SHUTDOWN` | Stop Caddy when hypeman shuts down (set to `true` for dev) | `false` | -| `ACME_EMAIL` | Email for ACME certificate registration (required for TLS ingresses) | _(empty)_ | -| `ACME_DNS_PROVIDER` | DNS provider for ACME challenges: `cloudflare` | _(empty)_ | -| `ACME_CA` | ACME CA URL (empty = Let's Encrypt production) | _(empty)_ | -| `TLS_ALLOWED_DOMAINS` | Comma-separated allowed domains for TLS (e.g., `*.example.com,api.other.com`) | _(empty)_ | -| `DNS_PROPAGATION_TIMEOUT` | Max time to wait for DNS propagation (e.g., `2m`) | _(empty)_ | -| `DNS_RESOLVERS` | Comma-separated DNS resolvers for propagation checking | _(empty)_ | -| `CLOUDFLARE_API_TOKEN` | Cloudflare API token (when using `cloudflare` provider) | _(empty)_ | -| `DOCKER_SOCKET` | Path to Docker socket (for builder image builds) | `/var/run/docker.sock` | +Hypeman reads configuration from a YAML config file. See `config.example.yaml` (Linux) and `config.example.darwin.yaml` (macOS) for all available settings with comments. + +The config file is searched in these locations (first match wins): +- Path specified by `CONFIG_PATH` environment variable +- `/etc/hypeman/config.yaml` (Linux) +- `~/.config/hypeman/config.yaml` (all platforms) + +Common settings: + +| Key | Description | Default | +|-----|-------------|---------| +| `port` | HTTP server port | `8080` | +| `data_dir` | Data directory for VM images, volumes, etc. | `/var/lib/hypeman` | +| `jwt_secret` | Secret key for JWT authentication (required) | _(empty)_ | +| `env` | Deployment environment (filters telemetry) | `unset` | +| `network.bridge_name` | Network bridge for VM networking | `vmbr0` | +| `network.subnet_cidr` | CIDR for the VM network subnet | `10.100.0.0/16` | +| `network.uplink_interface` | Host interface for VM internet access | _(auto-detect)_ | +| `network.dns_server` | DNS server for VMs | `1.1.1.1` | +| `caddy.listen_address` | Address for Caddy ingress listeners | `0.0.0.0` | +| `caddy.admin_address` | Address for Caddy admin API | `127.0.0.1` | +| `caddy.admin_port` | Port for Caddy admin API | `2019` | +| `caddy.stop_on_shutdown` | Stop Caddy when hypeman shuts down | `false` | +| `logging.level` | Log level (debug, info, warn, error) | `info` | +| `otel.enabled` | Enable OpenTelemetry traces/metrics | `false` | +| `otel.endpoint` | OTLP gRPC endpoint | `127.0.0.1:4317` | +| `limits.max_concurrent_builds` | Max concurrent image builds | `1` | +| `limits.max_overlay_size` | Max overlay filesystem size | `100GB` | +| `acme.email` | Email for ACME certificate registration | _(empty)_ | +| `acme.dns_provider` | DNS provider for ACME challenges | _(empty)_ | +| `acme.cloudflare_api_token` | Cloudflare API token | _(empty)_ | +| `build.docker_socket` | Path to Docker socket | `/var/run/docker.sock` | + +Environment variables can also override any config key using `__` as the nesting separator (e.g. `CADDY__LISTEN_ADDRESS` overrides `caddy.listen_address`). **Important: Subnet Configuration** The default subnet `10.100.0.0/16` is chosen to avoid common conflicts. Hypeman will detect conflicts with existing routes on startup and fail with guidance. -If you need a different subnet, set `SUBNET_CIDR` in your environment. The gateway is automatically derived as the first IP in the subnet (e.g., `10.100.0.0/16` → `10.100.0.1`). +If you need a different subnet, set `network.subnet_cidr` in your config file. The gateway is automatically derived as the first IP in the subnet (e.g., `10.100.0.0/16` → `10.100.0.1`). **Alternative subnets if needed:** @@ -136,14 +137,15 @@ If you need a different subnet, set `SUBNET_CIDR` in your environment. The gatew **Example:** -```bash -# In your .env file -SUBNET_CIDR=172.30.0.0/16 +```yaml +# In your config.yaml +network: + subnet_cidr: 172.30.0.0/16 ``` -**Finding the uplink interface (`UPLINK_INTERFACE`)** +**Finding the uplink interface (`network.uplink_interface`)** -`UPLINK_INTERFACE` tells Hypeman which host interface to use for routing VM traffic to the outside world (for iptables MASQUERADE rules). On many hosts this is `eth0`, but laptops and more complex setups often use Wi‑Fi or other names. +`network.uplink_interface` tells Hypeman which host interface to use for routing VM traffic to the outside world (for iptables MASQUERADE rules). On many hosts this is `eth0`, but laptops and more complex setups often use Wi-Fi or other names. **Quick way to discover it:** @@ -158,10 +160,11 @@ Look for the `dev` field in the output, for example: 1.1.1.1 via 192.168.12.1 dev wlp2s0 src 192.168.12.98 ``` -In this case, `wlp2s0` is the uplink interface, so you would set: +In this case, `wlp2s0` is the uplink interface, so you would set in your config file: -```bash -UPLINK_INTERFACE=wlp2s0 +```yaml +network: + uplink_interface: wlp2s0 ``` You can also inspect all routes: @@ -178,15 +181,13 @@ Hypeman uses Caddy with automatic ACME certificates for TLS termination. Certifi To enable TLS ingresses: -1. Configure ACME credentials in your `.env`: +1. Configure ACME credentials in your `config.yaml`: -```bash -# Required for any TLS ingress -ACME_EMAIL=admin@example.com - -# For Cloudflare -ACME_DNS_PROVIDER=cloudflare -CLOUDFLARE_API_TOKEN=your-api-token +```yaml +acme: + email: admin@example.com + dns_provider: cloudflare + cloudflare_api_token: your-api-token ``` 2. Create an ingress with TLS enabled: @@ -205,13 +206,13 @@ curl -X POST http://localhost:8080/v1/ingresses \ }' ``` -Certificates are stored in `$DATA_DIR/caddy/data/` and auto-renewed by Caddy. +Certificates are stored in `/caddy/data/` and auto-renewed by Caddy. ### Setup ```bash -cp .env.example .env -# Edit .env and set JWT_SECRET and other configuration values +cp config.example.yaml ~/.config/hypeman/config.yaml +# Edit config.yaml and set jwt_secret and other configuration values ``` ### Data directory @@ -253,15 +254,16 @@ make gen-jwt make dev ``` -The server will start on port 8080 (configurable via `PORT` environment variable). +The server will start on port 8080 (configurable via `port` in config.yaml). ### Setting Up the Builder Image (for Dockerfile builds) -The builder image is required for `hypeman build` to work. When `BUILDER_IMAGE` is unset or empty, the server will automatically build and push the builder image on startup using Docker. This is the easiest way to get started — just ensure Docker is available and run `make dev`. If a build is requested while the builder image is still being prepared, the server returns a clear error asking you to retry shortly. +The builder image is required for `hypeman build` to work. When `build.builder_image` is unset or empty, the server will automatically build and push the builder image on startup using Docker. This is the easiest way to get started — just ensure Docker is available and run `make dev`. If a build is requested while the builder image is still being prepared, the server returns a clear error asking you to retry shortly. -On macOS with Colima, set the Docker socket path: -```bash -DOCKER_SOCKET=$HOME/.colima/default/docker.sock +On macOS with Colima, set the Docker socket path in your config file: +```yaml +build: + docker_socket: ~/.colima/default/docker.sock ``` ### Local OpenTelemetry (optional) @@ -287,9 +289,11 @@ docker run -d --name lgtm \ # If developing on a remote server, forward the port to your local machine (or YOLO): # ssh -L 3001:localhost:3000 your-server (then open http://localhost:3001) -# Enable OTel in .env (set ENV to your name to filter your telemetry) -echo "OTEL_ENABLED=true" >> .env -echo "ENV=yourname" >> .env +# Enable OTel in config.yaml (set env to your name to filter your telemetry) +# Add to your config.yaml: +# otel: +# enabled: true +# env: yourname # Restart dev server make dev @@ -359,8 +363,9 @@ export PATH="/opt/homebrew/opt/e2fsprogs/bin:/opt/homebrew/opt/e2fsprogs/sbin:$P # Add to ~/.zshrc for persistence # 3. Configure environment -cp .env.darwin.example .env -# Edit .env as needed (defaults work for local development) +mkdir -p ~/.config/hypeman +cp config.example.darwin.yaml ~/.config/hypeman/config.yaml +# Edit config.yaml as needed (defaults work for local development) # 4. Create data directory mkdir -p ~/Library/Application\ Support/hypeman @@ -387,16 +392,16 @@ The `make dev` command automatically detects macOS and: ### macOS-Specific Configuration -The following environment variables work differently on macOS (see `.env.darwin.example`): +The following config keys work differently on macOS (see `config.example.darwin.yaml`): -| Variable | Linux | macOS | +| Config key | Linux | macOS | |----------|-------|-------| -| `DEFAULT_HYPERVISOR` | `cloud-hypervisor` | `vz` | -| `DATA_DIR` | `/var/lib/hypeman` | `~/Library/Application Support/hypeman` | -| `INTERNAL_DNS_PORT` | `5353` | `5354` (5353 is used by mDNSResponder) | -| `BRIDGE_NAME` | Used | Ignored (NAT) | -| `SUBNET_CIDR` | Used | Ignored (NAT) | -| `UPLINK_INTERFACE` | Used | Ignored (NAT) | +| `hypervisor.default` | `cloud-hypervisor` | `vz` | +| `data_dir` | `/var/lib/hypeman` | `~/Library/Application Support/hypeman` | +| `caddy.internal_dns_port` | `5353` | `5354` (5353 is used by mDNSResponder) | +| `network.bridge_name` | Used | Ignored (NAT) | +| `network.subnet_cidr` | Used | Ignored (NAT) | +| `network.uplink_interface` | Used | Ignored (NAT) | | Network rate limiting | Supported | Not supported | | GPU passthrough | Supported (VFIO) | Not supported | @@ -463,8 +468,8 @@ brew install caddy **"address already in use" on port 5353** - Port 5353 is used by mDNSResponder (Bonjour) on macOS -- Use port 5354 instead: `INTERNAL_DNS_PORT=5354` in `.env` -- The `.env.darwin.example` already has this configured correctly +- Use port 5354 instead: set `caddy.internal_dns_port: 5354` in `config.yaml` +- The `config.example.darwin.yaml` already has this configured correctly **"Virtualization.framework is not available"** - Ensure you're on macOS 11.0+ @@ -475,7 +480,7 @@ brew install caddy - Check: `sw_vers` and `uname -m` (should be arm64) **VM fails to start** -- Check serial log: `$DATA_DIR/instances//serial.log` +- Check serial log: `/instances//serial.log` - Ensure kernel and initrd paths are correct in config **IOMMU/VFIO warnings at startup** diff --git a/Makefile b/Makefile index 28e5ae9d..f3e1cf91 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ $(BIN_DIR): OAPI_CODEGEN ?= $(BIN_DIR)/oapi-codegen AIR ?= $(BIN_DIR)/air WIRE ?= $(BIN_DIR)/wire -GODOTENV ?= $(BIN_DIR)/godotenv XCADDY ?= $(BIN_DIR)/xcaddy # Install oapi-codegen @@ -26,15 +25,11 @@ $(AIR): | $(BIN_DIR) $(WIRE): | $(BIN_DIR) GOBIN=$(BIN_DIR) go install github.com/google/wire/cmd/wire@latest -# Install godotenv for loading .env files -$(GODOTENV): | $(BIN_DIR) - GOBIN=$(BIN_DIR) go install github.com/joho/godotenv/cmd/godotenv@latest - # Install xcaddy for building Caddy with plugins $(XCADDY): | $(BIN_DIR) GOBIN=$(BIN_DIR) go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest -install-tools: $(OAPI_CODEGEN) $(AIR) $(WIRE) $(GODOTENV) $(XCADDY) +install-tools: $(OAPI_CODEGEN) $(AIR) $(WIRE) $(XCADDY) # Download Cloud Hypervisor binaries download-ch-binaries: @@ -261,8 +256,9 @@ test-darwin: build-embedded sign-vz-shim # Generate JWT token for testing # Usage: make gen-jwt [USER_ID=test-user] -gen-jwt: $(GODOTENV) - @$(GODOTENV) -f .env go run ./cmd/gen-jwt -user-id $${USER_ID:-test-user} +# Checks CONFIG_PATH, then local config.yaml, then default config paths +gen-jwt: + @CONFIG_PATH=$${CONFIG_PATH:-$$([ -f config.yaml ] && echo config.yaml)} go run ./cmd/gen-jwt -user-id $${USER_ID:-test-user} # Build the generic builder image for builds build-builder: diff --git a/README.md b/README.md index 93de5d51..3a1a449f 100644 --- a/README.md +++ b/README.md @@ -35,29 +35,59 @@ Install Hypeman (Linux and macOS supported): curl -fsSL https://get.hypeman.sh | bash ``` -This installs both the Hypeman server and CLI. The installer handles all dependencies and configuration automatically. +This installs the Hypeman server, CLI, and token tool. The installer: +- Generates a YAML config file with a random JWT secret +- Starts the server as a system service (launchd on macOS, systemd on Linux) +- Creates a CLI config file (`~/.config/hypeman/cli.yaml`) with a pre-authenticated token -## CLI Installation +No environment variables needed -- just run `hypeman` commands immediately after install. -To use Hypeman via the CLI on a separate machine: +## Remote CLI Access -**Homebrew:** +To use the Hypeman CLI from a **different machine** than the server: + +**Homebrew (macOS):** ```bash brew install kernel/tap/hypeman ``` +**Linux:** +```bash +curl -fsSL https://get.hypeman.sh/cli | bash +``` + **Go:** ```bash go install 'github.com/kernel/hypeman-cli/cmd/hypeman@latest' ``` -**Configure CLI access:** +Then create a CLI config file at `~/.config/hypeman/cli.yaml`: + +```yaml +base_url: http://:8080 +api_key: "" +``` + +To generate a token, run `hypeman-token` on the server: ```bash -export HYPEMAN_API_KEY="" -export HYPEMAN_BASE_URL="http://:8080" +hypeman-token -user-id "my-user" -duration 8760h ``` +Environment variables (`HYPEMAN_BASE_URL`, `HYPEMAN_API_KEY`) and CLI flags (`--base-url`) also work and take precedence over the config file. + +## Configuration + +Hypeman is configured via YAML config files. + +| Component | Config File | +|-----------|-------------| +| Server | `/etc/hypeman/config.yaml` (Linux) or `~/.config/hypeman/config.yaml` (macOS) | +| CLI | `~/.config/hypeman/cli.yaml` | + + +See [`config.example.yaml`](config.example.yaml) (Linux) and [`config.example.darwin.yaml`](config.example.darwin.yaml) (macOS) for all available server options. + ## Usage ```bash diff --git a/cli.example.yaml b/cli.example.yaml new file mode 100644 index 00000000..c0007e63 --- /dev/null +++ b/cli.example.yaml @@ -0,0 +1,19 @@ +# ============================================================================= +# Hypeman CLI Configuration +# ============================================================================= +# Place this file at ~/.config/hypeman/cli.yaml +# +# The install script automatically generates this file with a valid token +# for local access. For remote CLI access, create this file manually. +# +# Configuration precedence (highest to lowest): +# 1. CLI flags (e.g., --base-url) +# 2. Environment variables (HYPEMAN_BASE_URL, HYPEMAN_API_KEY) +# 3. This YAML config file +# ============================================================================= + +# Hypeman API URL +base_url: http://localhost:8080 + +# API authentication token (generated by hypeman-token) +api_key: "" diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index af71d365..310c538a 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -25,10 +25,12 @@ import ( // newTestService creates an ApiService for testing with automatic cleanup func newTestService(t *testing.T) *ApiService { cfg := &config.Config{ - DataDir: t.TempDir(), - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: t.TempDir(), + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } p := paths.New(cfg.DataDir) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 12b22318..4a53434c 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -3,10 +3,16 @@ package config import ( "fmt" "os" + "path/filepath" + "runtime" "runtime/debug" - "strconv" + "strings" - "github.com/joho/godotenv" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/env" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/structs" + "github.com/knadh/koanf/v2" ) func getHostname() string { @@ -49,258 +55,349 @@ func getBuildVersion() string { return revision } -type Config struct { - Port string - DataDir string - BridgeName string - SubnetCIDR string - SubnetGateway string - UplinkInterface string - JwtSecret string - DNSServer string - MaxConcurrentBuilds int - MaxOverlaySize string - LogMaxSize string - LogMaxFiles int - LogRotateInterval string - - // Resource limits - per instance - MaxVcpusPerInstance int // Max vCPUs for a single VM (0 = unlimited) - MaxMemoryPerInstance string // Max memory for a single VM (0 = unlimited) - - // Resource limits - aggregate - // Note: CPU/memory aggregate limits are now handled via oversubscription ratios (OVERSUB_CPU, OVERSUB_MEMORY) - MaxTotalVolumeStorage string // Total volume storage limit (0 = unlimited) - - // OpenTelemetry configuration - OtelEnabled bool // Enable OpenTelemetry - OtelEndpoint string // OTLP endpoint (gRPC) - OtelServiceName string // Service name for tracing - OtelServiceInstanceID string // Service instance ID (default: hostname) - OtelInsecure bool // Disable TLS for OTLP - Version string // Application version for telemetry - Env string // Deployment environment (e.g., dev, staging, prod) - - // Logging configuration - LogLevel string // Default log level (debug, info, warn, error) - - // Caddy / Ingress configuration - CaddyListenAddress string // Address for Caddy to listen on - CaddyAdminAddress string // Address for Caddy admin API - CaddyAdminPort int // Port for Caddy admin API - InternalDNSPort int // Port for internal DNS server (used for dynamic upstreams) - CaddyStopOnShutdown bool // Stop Caddy when hypeman shuts down - - // ACME / TLS configuration - AcmeEmail string // ACME account email (required for TLS ingresses) - AcmeDnsProvider string // DNS provider: "cloudflare" - AcmeCA string // ACME CA URL (empty = Let's Encrypt production) - DnsPropagationTimeout string // Max time to wait for DNS propagation (e.g., "2m") - DnsResolvers string // Comma-separated DNS resolvers for propagation checking - TlsAllowedDomains string // Comma-separated list of allowed domain patterns for TLS (e.g., "*.example.com,api.example.com") - - // Cloudflare configuration (if AcmeDnsProvider=cloudflare) - CloudflareApiToken string // Cloudflare API token - - // API ingress configuration - exposes Hypeman API via Caddy - ApiHostname string // Hostname for API access (e.g., hypeman.hostname.kernel.sh). Empty = disabled. - ApiTLS bool // Enable TLS for API hostname - ApiRedirectHTTP bool // Redirect HTTP to HTTPS for API hostname - - // Build system configuration - MaxConcurrentSourceBuilds int // Max concurrent source-to-image builds - BuilderImage string // OCI image for builder VMs - RegistryURL string // URL of registry for built images - RegistryInsecure bool // Skip TLS verification for registry (for self-signed certs) - RegistryCACertFile string // Path to CA certificate file for registry TLS verification - BuildTimeout int // Default build timeout in seconds - BuildSecretsDir string // Directory containing build secrets (optional) - DockerSocket string // Path to Docker socket (for building builder image) - - // Hypervisor configuration - DefaultHypervisor string // Default hypervisor type: "cloud-hypervisor" or "qemu" - - // GPU configuration - GPUProfileCacheTTL string // TTL for GPU profile metadata cache (e.g., "30m") - - // Oversubscription ratios (1.0 = no oversubscription, 2.0 = 2x oversubscription) - OversubCPU float64 // CPU oversubscription ratio - OversubMemory float64 // Memory oversubscription ratio - OversubDisk float64 // Disk oversubscription ratio - OversubNetwork float64 // Network oversubscription ratio - OversubDiskIO float64 // Disk I/O oversubscription ratio - - // Network rate limiting - UploadBurstMultiplier int // Multiplier for upload burst ceiling vs guaranteed rate (default: 4) - DownloadBurstMultiplier int // Multiplier for download burst bucket vs rate (default: 4) - - // Resource capacity limits (empty = auto-detect from host) - DiskLimit string // Hard disk limit for DataDir, e.g. "500GB" - NetworkLimit string // Hard network limit, e.g. "10Gbps" (empty = detect from uplink speed) - DiskIOLimit string // Hard disk I/O limit, e.g. "500MB/s" (empty = auto-detect from disk type) - MaxImageStorage float64 // Max image storage as fraction of disk (0.2 = 20%), counts OCI cache + rootfs +// NetworkConfig holds network bridge and interface settings. +type NetworkConfig struct { + BridgeName string `koanf:"bridge_name"` + SubnetCIDR string `koanf:"subnet_cidr"` + SubnetGateway string `koanf:"subnet_gateway"` + UplinkInterface string `koanf:"uplink_interface"` + DNSServer string `koanf:"dns_server"` + UploadBurstMultiplier int `koanf:"upload_burst_multiplier"` + DownloadBurstMultiplier int `koanf:"download_burst_multiplier"` } -// Load loads configuration from environment variables -// Automatically loads .env file if present -func Load() *Config { - // Try to load .env file (fail silently if not present) - _ = godotenv.Load() - - cfg := &Config{ - Port: getEnv("PORT", "8080"), - DataDir: getEnv("DATA_DIR", "/var/lib/hypeman"), - BridgeName: getEnv("BRIDGE_NAME", "vmbr0"), - SubnetCIDR: getEnv("SUBNET_CIDR", "10.100.0.0/16"), - SubnetGateway: getEnv("SUBNET_GATEWAY", ""), // empty = derived as first IP from subnet - UplinkInterface: getEnv("UPLINK_INTERFACE", ""), // empty = auto-detect from default route - JwtSecret: getEnv("JWT_SECRET", ""), - DNSServer: getEnv("DNS_SERVER", "1.1.1.1"), - MaxConcurrentBuilds: getEnvInt("MAX_CONCURRENT_BUILDS", 1), - MaxOverlaySize: getEnv("MAX_OVERLAY_SIZE", "100GB"), - LogMaxSize: getEnv("LOG_MAX_SIZE", "50MB"), - LogMaxFiles: getEnvInt("LOG_MAX_FILES", 1), - LogRotateInterval: getEnv("LOG_ROTATE_INTERVAL", "5m"), - - // Resource limits - per instance (0 = unlimited) - MaxVcpusPerInstance: getEnvInt("MAX_VCPUS_PER_INSTANCE", 16), - MaxMemoryPerInstance: getEnv("MAX_MEMORY_PER_INSTANCE", "32GB"), - - // Resource limits - aggregate - // Note: CPU/memory aggregate limits are now handled via oversubscription ratios - MaxTotalVolumeStorage: getEnv("MAX_TOTAL_VOLUME_STORAGE", ""), - - // OpenTelemetry configuration - OtelEnabled: getEnvBool("OTEL_ENABLED", false), - OtelEndpoint: getEnv("OTEL_ENDPOINT", "127.0.0.1:4317"), - OtelServiceName: getEnv("OTEL_SERVICE_NAME", "hypeman"), - OtelServiceInstanceID: getEnv("OTEL_SERVICE_INSTANCE_ID", getHostname()), - OtelInsecure: getEnvBool("OTEL_INSECURE", true), - Version: getEnv("VERSION", getBuildVersion()), - Env: getEnv("ENV", "unset"), - - // Logging configuration - LogLevel: getEnv("LOG_LEVEL", "info"), - - // Caddy / Ingress configuration - CaddyListenAddress: getEnv("CADDY_LISTEN_ADDRESS", "0.0.0.0"), - CaddyAdminAddress: getEnv("CADDY_ADMIN_ADDRESS", "127.0.0.1"), - CaddyAdminPort: getEnvInt("CADDY_ADMIN_PORT", 0), // 0 = random port to prevent conflicts on shared dev machines - InternalDNSPort: getEnvInt("INTERNAL_DNS_PORT", 0), // 0 = random port; used for dynamic upstream resolution - // Set to false if you're likely to frequently update hypeman - CaddyStopOnShutdown: getEnvBool("CADDY_STOP_ON_SHUTDOWN", true), - - // ACME / TLS configuration - AcmeEmail: getEnv("ACME_EMAIL", ""), - AcmeDnsProvider: getEnv("ACME_DNS_PROVIDER", ""), - AcmeCA: getEnv("ACME_CA", ""), - DnsPropagationTimeout: getEnv("DNS_PROPAGATION_TIMEOUT", ""), - DnsResolvers: getEnv("DNS_RESOLVERS", ""), - TlsAllowedDomains: getEnv("TLS_ALLOWED_DOMAINS", ""), // Empty = no TLS domains allowed - - // Cloudflare configuration - CloudflareApiToken: getEnv("CLOUDFLARE_API_TOKEN", ""), - - // API ingress configuration - ApiHostname: getEnv("API_HOSTNAME", ""), // Empty = disabled - ApiTLS: getEnvBool("API_TLS", true), // Default to TLS enabled - ApiRedirectHTTP: getEnvBool("API_REDIRECT_HTTP", true), - - // Build system configuration - MaxConcurrentSourceBuilds: getEnvInt("MAX_CONCURRENT_SOURCE_BUILDS", 2), - BuilderImage: getEnv("BUILDER_IMAGE", "hypeman/builder:latest"), - RegistryURL: getEnv("REGISTRY_URL", "localhost:8080"), - RegistryInsecure: getEnvBool("REGISTRY_INSECURE", false), - RegistryCACertFile: getEnv("REGISTRY_CA_CERT_FILE", ""), // Path to CA cert for registry TLS - BuildTimeout: getEnvInt("BUILD_TIMEOUT", 600), - BuildSecretsDir: getEnv("BUILD_SECRETS_DIR", ""), // Optional: path to directory with build secrets - DockerSocket: getEnv("DOCKER_SOCKET", "/var/run/docker.sock"), - - // Hypervisor configuration - DefaultHypervisor: getEnv("DEFAULT_HYPERVISOR", "cloud-hypervisor"), - - // GPU configuration - GPUProfileCacheTTL: getEnv("GPU_PROFILE_CACHE_TTL", "30m"), - - // Oversubscription ratios (1.0 = no oversubscription) - OversubCPU: getEnvFloat("OVERSUB_CPU", 4.0), - OversubMemory: getEnvFloat("OVERSUB_MEMORY", 1.0), - OversubDisk: getEnvFloat("OVERSUB_DISK", 1.0), - OversubNetwork: getEnvFloat("OVERSUB_NETWORK", 2.0), - OversubDiskIO: getEnvFloat("OVERSUB_DISK_IO", 2.0), - - // Network rate limiting - UploadBurstMultiplier: getEnvInt("UPLOAD_BURST_MULTIPLIER", 4), - DownloadBurstMultiplier: getEnvInt("DOWNLOAD_BURST_MULTIPLIER", 4), - - // Resource capacity limits (empty = auto-detect) - DiskLimit: getEnv("DISK_LIMIT", ""), - NetworkLimit: getEnv("NETWORK_LIMIT", ""), - DiskIOLimit: getEnv("DISK_IO_LIMIT", ""), - MaxImageStorage: getEnvFloat("MAX_IMAGE_STORAGE", 0.2), // 20% of disk by default - } +// CaddyConfig holds Caddy reverse-proxy / ingress settings. +type CaddyConfig struct { + ListenAddress string `koanf:"listen_address"` + AdminAddress string `koanf:"admin_address"` + AdminPort int `koanf:"admin_port"` + InternalDNSPort int `koanf:"internal_dns_port"` + StopOnShutdown bool `koanf:"stop_on_shutdown"` +} - return cfg +// ACMEConfig holds ACME / TLS certificate settings. +type ACMEConfig struct { + Email string `koanf:"email"` + DNSProvider string `koanf:"dns_provider"` + CA string `koanf:"ca"` + DNSPropagationTimeout string `koanf:"dns_propagation_timeout"` + DNSResolvers string `koanf:"dns_resolvers"` + AllowedDomains string `koanf:"allowed_domains"` + CloudflareAPIToken string `koanf:"cloudflare_api_token"` } -func getEnv(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue +// APIConfig holds API ingress settings (exposes Hypeman API via Caddy). +type APIConfig struct { + Hostname string `koanf:"hostname"` + TLS bool `koanf:"tls"` + RedirectHTTP bool `koanf:"redirect_http"` +} + +// OtelConfig holds OpenTelemetry settings. +type OtelConfig struct { + Enabled bool `koanf:"enabled"` + Endpoint string `koanf:"endpoint"` + ServiceName string `koanf:"service_name"` + ServiceInstanceID string `koanf:"service_instance_id"` + Insecure bool `koanf:"insecure"` +} + +// LoggingConfig holds log rotation and level settings. +type LoggingConfig struct { + Level string `koanf:"level"` + MaxSize string `koanf:"max_size"` + MaxFiles int `koanf:"max_files"` + RotateInterval string `koanf:"rotate_interval"` +} + +// BuildConfig holds source-to-image build system settings. +type BuildConfig struct { + MaxConcurrentSourceBuilds int `koanf:"max_concurrent_source_builds"` + BuilderImage string `koanf:"builder_image"` + Timeout int `koanf:"timeout"` + SecretsDir string `koanf:"secrets_dir"` + DockerSocket string `koanf:"docker_socket"` +} + +// RegistryConfig holds OCI registry settings. +type RegistryConfig struct { + URL string `koanf:"url"` + Insecure bool `koanf:"insecure"` + CACertFile string `koanf:"ca_cert_file"` +} + +// LimitsConfig holds per-instance and aggregate resource limits. +type LimitsConfig struct { + MaxVcpusPerInstance int `koanf:"max_vcpus_per_instance"` + MaxMemoryPerInstance string `koanf:"max_memory_per_instance"` + MaxTotalVolumeStorage string `koanf:"max_total_volume_storage"` + MaxConcurrentBuilds int `koanf:"max_concurrent_builds"` + MaxOverlaySize string `koanf:"max_overlay_size"` + MaxImageStorage float64 `koanf:"max_image_storage"` +} + +// OversubscriptionConfig holds oversubscription ratios (1.0 = no oversubscription). +type OversubscriptionConfig struct { + CPU float64 `koanf:"cpu"` + Memory float64 `koanf:"memory"` + Disk float64 `koanf:"disk"` + Network float64 `koanf:"network"` + DiskIO float64 `koanf:"disk_io"` +} + +// CapacityConfig holds hard resource capacity limits (empty = auto-detect from host). +type CapacityConfig struct { + Disk string `koanf:"disk"` + Network string `koanf:"network"` + DiskIO string `koanf:"disk_io"` +} + +// HypervisorConfig holds hypervisor settings. +type HypervisorConfig struct { + Default string `koanf:"default"` +} + +// GPUConfig holds GPU-related settings. +type GPUConfig struct { + ProfileCacheTTL string `koanf:"profile_cache_ttl"` +} + +// Config is the top-level Hypeman server configuration. +type Config struct { + Port string `koanf:"port"` + DataDir string `koanf:"data_dir"` + JwtSecret string `koanf:"jwt_secret"` + Env string `koanf:"env"` + Version string `koanf:"version"` + + Network NetworkConfig `koanf:"network"` + Caddy CaddyConfig `koanf:"caddy"` + ACME ACMEConfig `koanf:"acme"` + API APIConfig `koanf:"api"` + Otel OtelConfig `koanf:"otel"` + Logging LoggingConfig `koanf:"logging"` + Build BuildConfig `koanf:"build"` + Registry RegistryConfig `koanf:"registry"` + Limits LimitsConfig `koanf:"limits"` + Oversubscription OversubscriptionConfig `koanf:"oversubscription"` + Capacity CapacityConfig `koanf:"capacity"` + Hypervisor HypervisorConfig `koanf:"hypervisor"` + GPU GPUConfig `koanf:"gpu"` } -func getEnvInt(key string, defaultValue int) int { - if value := os.Getenv(key); value != "" { - if intVal, err := strconv.Atoi(value); err == nil { - return intVal +// GetDefaultConfigPaths returns the default config file paths to search. +// Returns paths in order of precedence (first found wins). +func GetDefaultConfigPaths() []string { + home, _ := os.UserHomeDir() + if runtime.GOOS == "darwin" { + return []string{ + filepath.Join(home, ".config", "hypeman", "config.yaml"), } } - return defaultValue + // Linux: check /etc first, then user config + return []string{ + "/etc/hypeman/config.yaml", + filepath.Join(home, ".config", "hypeman", "config.yaml"), + } } -func getEnvBool(key string, defaultValue bool) bool { - if value := os.Getenv(key); value != "" { - if boolVal, err := strconv.ParseBool(value); err == nil { - return boolVal - } + +// defaultConfig returns a Config struct with all default values set. +func defaultConfig() *Config { + return &Config{ + Port: "8080", + DataDir: "/var/lib/hypeman", + JwtSecret: "", + Env: "unset", + Version: getBuildVersion(), + + Network: NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + SubnetGateway: "", + UplinkInterface: "", + DNSServer: "1.1.1.1", + UploadBurstMultiplier: 4, + DownloadBurstMultiplier: 4, + }, + + Caddy: CaddyConfig{ + ListenAddress: "0.0.0.0", + AdminAddress: "127.0.0.1", + AdminPort: 0, + InternalDNSPort: 0, + StopOnShutdown: true, + }, + + ACME: ACMEConfig{ + Email: "", + DNSProvider: "", + CA: "", + DNSPropagationTimeout: "", + DNSResolvers: "", + AllowedDomains: "", + CloudflareAPIToken: "", + }, + + API: APIConfig{ + Hostname: "", + TLS: true, + RedirectHTTP: true, + }, + + Otel: OtelConfig{ + Enabled: false, + Endpoint: "127.0.0.1:4317", + ServiceName: "hypeman", + ServiceInstanceID: getHostname(), + Insecure: true, + }, + + Logging: LoggingConfig{ + Level: "info", + MaxSize: "50MB", + MaxFiles: 1, + RotateInterval: "5m", + }, + + Build: BuildConfig{ + MaxConcurrentSourceBuilds: 2, + BuilderImage: "", // empty = build from embedded Dockerfile on first run + Timeout: 600, + SecretsDir: "", + DockerSocket: "/var/run/docker.sock", + }, + + Registry: RegistryConfig{ + URL: "localhost:8080", + Insecure: false, + CACertFile: "", + }, + + Limits: LimitsConfig{ + MaxVcpusPerInstance: 16, + MaxMemoryPerInstance: "32GB", + MaxTotalVolumeStorage: "", + MaxConcurrentBuilds: 1, + MaxOverlaySize: "100GB", + MaxImageStorage: 0.2, + }, + + Oversubscription: OversubscriptionConfig{ + CPU: 4.0, + Memory: 1.0, + Disk: 1.0, + Network: 2.0, + DiskIO: 2.0, + }, + + Capacity: CapacityConfig{ + Disk: "", + Network: "", + DiskIO: "", + }, + + Hypervisor: HypervisorConfig{ + Default: "cloud-hypervisor", + }, + + GPU: GPUConfig{ + ProfileCacheTTL: "30m", + }, } - return defaultValue } -func getEnvFloat(key string, defaultValue float64) float64 { - if value := os.Getenv(key); value != "" { - if floatVal, err := strconv.ParseFloat(value, 64); err == nil { - return floatVal +// Load loads configuration with the following precedence (highest to lowest): +// +// 1. Environment variables — uses double-underscore (__) as the nesting +// separator: PORT, DATA_DIR, JWT_SECRET for top-level keys and +// CADDY__LISTEN_ADDRESS, NETWORK__BRIDGE_NAME, etc. for nested keys. +// 2. YAML config file (if found) +// 3. Default values +// +// The configPath parameter specifies an explicit config file path. +// If empty, searches default locations based on OS. +// Returns an error if an explicitly provided configPath cannot be loaded. +func Load(configPath string) (*Config, error) { + k := koanf.New(".") + + // 1. Load defaults first + defaults := defaultConfig() + if err := k.Load(structs.Provider(defaults, "koanf"), nil); err != nil { + return nil, fmt.Errorf("failed to load default config: %w", err) + } + + // 2. Load from YAML config file + explicitPath := configPath != "" + if !explicitPath { + // Search default paths (best-effort, file may not exist) + for _, path := range GetDefaultConfigPaths() { + if _, err := os.Stat(path); err == nil { + configPath = path + break + } + } + } + if configPath != "" { + if err := k.Load(file.Provider(configPath), yaml.Parser()); err != nil { + if explicitPath { + // Explicit path must be loadable + return nil, fmt.Errorf("failed to load config from %s: %w", configPath, err) + } + // Auto-discovered path failed — continue with defaults + env } } - return defaultValue + + // 3. Overlay environment variables (highest precedence) + // The "__" delimiter maps double-underscore in env var names to nested + // koanf key separators: CADDY__LISTEN_ADDRESS → caddy.listen_address. + // Single underscores are preserved: JWT_SECRET → jwt_secret (top-level). + envProvider := env.ProviderWithValue("", "__", func(key string, value string) (string, interface{}) { + if value == "" { + return "", nil + } + return strings.ToLower(key), value + }) + _ = k.Load(envProvider, nil) + + // 4. Unmarshal to Config struct + var cfg Config + if err := k.Unmarshal("", &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) + } + + return &cfg, nil } // Validate checks configuration values for correctness. // Returns an error if any configuration value is invalid. func (c *Config) Validate() error { - // Validate oversubscription ratios are positive - if c.OversubCPU <= 0 { - return fmt.Errorf("OVERSUB_CPU must be positive, got %v", c.OversubCPU) + if c.Oversubscription.CPU <= 0 { + return fmt.Errorf("oversubscription.cpu must be positive, got %v", c.Oversubscription.CPU) + } + if c.Oversubscription.Memory <= 0 { + return fmt.Errorf("oversubscription.memory must be positive, got %v", c.Oversubscription.Memory) + } + if c.Oversubscription.Disk <= 0 { + return fmt.Errorf("oversubscription.disk must be positive, got %v", c.Oversubscription.Disk) } - if c.OversubMemory <= 0 { - return fmt.Errorf("OVERSUB_MEMORY must be positive, got %v", c.OversubMemory) + if c.Oversubscription.Network <= 0 { + return fmt.Errorf("oversubscription.network must be positive, got %v", c.Oversubscription.Network) } - if c.OversubDisk <= 0 { - return fmt.Errorf("OVERSUB_DISK must be positive, got %v", c.OversubDisk) + if c.Oversubscription.DiskIO <= 0 { + return fmt.Errorf("oversubscription.disk_io must be positive, got %v", c.Oversubscription.DiskIO) } - if c.OversubNetwork <= 0 { - return fmt.Errorf("OVERSUB_NETWORK must be positive, got %v", c.OversubNetwork) + if c.Network.UploadBurstMultiplier < 1 { + return fmt.Errorf("network.upload_burst_multiplier must be >= 1, got %v", c.Network.UploadBurstMultiplier) } - if c.OversubDiskIO <= 0 { - return fmt.Errorf("OVERSUB_DISK_IO must be positive, got %v", c.OversubDiskIO) + if c.Network.DownloadBurstMultiplier < 1 { + return fmt.Errorf("network.download_burst_multiplier must be >= 1, got %v", c.Network.DownloadBurstMultiplier) } - if c.UploadBurstMultiplier < 1 { - return fmt.Errorf("UPLOAD_BURST_MULTIPLIER must be >= 1, got %v", c.UploadBurstMultiplier) + if c.Build.MaxConcurrentSourceBuilds <= 0 { + return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } - if c.DownloadBurstMultiplier < 1 { - return fmt.Errorf("DOWNLOAD_BURST_MULTIPLIER must be >= 1, got %v", c.DownloadBurstMultiplier) + if c.Build.Timeout <= 0 { + return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } return nil } diff --git a/cmd/api/main.go b/cmd/api/main.go index 561a9f3c..8d3c7cd6 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -45,7 +45,12 @@ func main() { func run() error { // Load config early for OTel initialization - cfg := config.Load() + // Config path can be specified via CONFIG_PATH env var or defaults to platform-specific locations + configPath := os.Getenv("CONFIG_PATH") + cfg, err := config.Load(configPath) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } // Validate configuration before proceeding if err := cfg.Validate(); err != nil { @@ -53,15 +58,15 @@ func run() error { } // Configure GPU profile cache TTL - devices.SetGPUProfileCacheTTL(cfg.GPUProfileCacheTTL) + devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ - Enabled: cfg.OtelEnabled, - Endpoint: cfg.OtelEndpoint, - ServiceName: cfg.OtelServiceName, - ServiceInstanceID: cfg.OtelServiceInstanceID, - Insecure: cfg.OtelInsecure, + Enabled: cfg.Otel.Enabled, + Endpoint: cfg.Otel.Endpoint, + ServiceName: cfg.Otel.ServiceName, + ServiceInstanceID: cfg.Otel.ServiceInstanceID, + Insecure: cfg.Otel.Insecure, Version: cfg.Version, Env: cfg.Env, } @@ -121,8 +126,8 @@ func run() error { logger := app.Logger // Log OTel status - if cfg.OtelEnabled { - logger.Info("OpenTelemetry enabled", "endpoint", cfg.OtelEndpoint, "service", cfg.OtelServiceName) + if cfg.Otel.Enabled { + logger.Info("OpenTelemetry enabled", "endpoint", cfg.Otel.Endpoint, "service", cfg.Otel.ServiceName) } // Validate JWT secret is configured @@ -143,12 +148,12 @@ func run() error { // Validate log rotation config var logMaxSize datasize.ByteSize - if err := logMaxSize.UnmarshalText([]byte(app.Config.LogMaxSize)); err != nil { - return fmt.Errorf("invalid LOG_MAX_SIZE %q: %w", app.Config.LogMaxSize, err) + if err := logMaxSize.UnmarshalText([]byte(app.Config.Logging.MaxSize)); err != nil { + return fmt.Errorf("invalid LOG_MAX_SIZE %q: %w", app.Config.Logging.MaxSize, err) } - logRotateInterval, err := time.ParseDuration(app.Config.LogRotateInterval) + logRotateInterval, err := time.ParseDuration(app.Config.Logging.RotateInterval) if err != nil { - return fmt.Errorf("invalid LOG_ROTATE_INTERVAL %q: %w", app.Config.LogRotateInterval, err) + return fmt.Errorf("invalid LOG_ROTATE_INTERVAL %q: %w", app.Config.Logging.RotateInterval, err) } // Ensure system files (kernel, initrd) exist before starting server @@ -237,7 +242,7 @@ func run() error { logger.Error("failed to initialize ingress manager", "error", err) return fmt.Errorf("initialize ingress manager: %w", err) } - logger.Info("Ingress manager initialized", "listen_addr", cfg.CaddyListenAddress, "admin", app.IngressManager.AdminURL()) + logger.Info("Ingress manager initialized", "listen_addr", cfg.Caddy.ListenAddress, "admin", app.IngressManager.AdminURL()) // Create router r := chi.NewRouter() @@ -321,8 +326,8 @@ func run() error { r.Use(middleware.Recoverer) // OpenTelemetry tracing middleware FIRST (creates span context) - if cfg.OtelEnabled { - r.Use(otelchi.Middleware(cfg.OtelServiceName, otelchi.WithChiRoutes(r))) + if cfg.Otel.Enabled { + r.Use(otelchi.Middleware(cfg.Otel.ServiceName, otelchi.WithChiRoutes(r))) } // Inject logger into request context for handlers to use @@ -445,16 +450,16 @@ func run() error { ticker := time.NewTicker(logRotateInterval) defer ticker.Stop() - logger.Info("log rotation scheduler started", "interval", app.Config.LogRotateInterval, "max_size", logMaxSize, "max_files", app.Config.LogMaxFiles) + logger.Info("log rotation scheduler started", "interval", app.Config.Logging.RotateInterval, "max_size", logMaxSize, "max_files", app.Config.Logging.MaxFiles) for { select { case <-gctx.Done(): return nil case <-ticker.C: - if err := app.InstanceManager.RotateLogs(gctx, int64(logMaxSize), app.Config.LogMaxFiles); err != nil { + if err := app.InstanceManager.RotateLogs(gctx, int64(logMaxSize), app.Config.Logging.MaxFiles); err != nil { logger.Error("log rotation failed", "error", err) } else { - logger.Info("log rotation completed", "max_size", logMaxSize, "max_files", app.Config.LogMaxFiles) + logger.Info("log rotation completed", "max_size", logMaxSize, "max_files", app.Config.Logging.MaxFiles) } } } diff --git a/cmd/gen-jwt/main.go b/cmd/gen-jwt/main.go index a14cd409..24a71f0f 100644 --- a/cmd/gen-jwt/main.go +++ b/cmd/gen-jwt/main.go @@ -7,21 +7,63 @@ import ( "time" "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/v2" ) +// getJWTSecret retrieves the JWT secret with the following precedence: +// 1. JWT_SECRET environment variable +// 2. jwt_secret from CONFIG_PATH config file (if set) +// 3. jwt_secret from default config.yaml paths (same as hypeman-api) +func getJWTSecret() string { + // 1. Check environment variable first (highest precedence) + if s := os.Getenv("JWT_SECRET"); s != "" { + return s + } + + // 2. Try CONFIG_PATH first (same as hypeman-api), then default paths + k := koanf.New(".") + if configPath := os.Getenv("CONFIG_PATH"); configPath != "" { + if err := k.Load(file.Provider(configPath), yaml.Parser()); err == nil { + if s := k.String("jwt_secret"); s != "" { + return s + } + } + } + + // 3. Try default config file paths + for _, path := range config.GetDefaultConfigPaths() { + if err := k.Load(file.Provider(path), yaml.Parser()); err == nil { + if s := k.String("jwt_secret"); s != "" { + return s + } + } + } + + return "" +} + func main() { - jwtSecret := os.Getenv("JWT_SECRET") + userID := flag.String("user-id", "test-user", "User ID to include in the JWT token") + duration := flag.Duration("duration", 24*time.Hour, "Token validity duration (e.g., 24h, 720h, 8760h)") + flag.Parse() + + jwtSecret := getJWTSecret() if jwtSecret == "" { - fmt.Fprintf(os.Stderr, "Error: JWT_SECRET environment variable is not set\n") + fmt.Fprintf(os.Stderr, "Error: JWT_SECRET not found.\n") + fmt.Fprintf(os.Stderr, "Set JWT_SECRET environment variable, set CONFIG_PATH, or ensure jwt_secret is configured in:\n") + for _, path := range config.GetDefaultConfigPaths() { + fmt.Fprintf(os.Stderr, " - %s\n", path) + } os.Exit(1) } - userID := flag.String("user-id", "test-user", "User ID to include in the JWT token") - flag.Parse() claims := jwt.MapClaims{ "sub": *userID, "iat": time.Now().Unix(), - "exp": time.Now().Add(24 * time.Hour).Unix(), + "exp": time.Now().Add(*duration).Unix(), } token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) tokenString, err := token.SignedString([]byte(jwtSecret)) diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml new file mode 100644 index 00000000..d491e493 --- /dev/null +++ b/config.example.darwin.yaml @@ -0,0 +1,132 @@ +# ============================================================================= +# macOS (Darwin) Configuration for Hypeman +# ============================================================================= +# Copy this file to ~/.config/hypeman/config.yaml and customize for your +# environment. +# +# Configuration precedence (highest to lowest): +# 1. Environment variables (e.g., JWT_SECRET, PORT) +# 2. This YAML config file +# 3. Built-in defaults +# +# Key differences from Linux (see config.example.yaml): +# - hypervisor.default: Use "vz" (Virtualization.framework) instead of cloud-hypervisor/qemu +# - data_dir: Uses macOS conventions (~/Library/Application Support) +# - Network settings: network.bridge_name, subnet_cidr, etc. are IGNORED (vz uses NAT) +# - Rate limiting: Not supported on macOS (no tc/HTB equivalent) +# - GPU passthrough: Not supported on macOS +# ============================================================================= + +# Required - used to sign and verify API tokens +jwt_secret: dev-secret-change-me + +# Data directory - use macOS conventions +# Note: ~ is NOT expanded; use full path or let the install script fill this in +data_dir: ~/Library/Application Support/hypeman + +# Server configuration +port: "8080" + +# ============================================================================= +# Hypervisor Configuration (IMPORTANT FOR MACOS) +# ============================================================================= +# On macOS, use "vz" (Virtualization.framework) +# - "cloud-hypervisor" and "qemu" are NOT supported on macOS +hypervisor: + default: vz + +# ============================================================================= +# Network Configuration (DIFFERENT ON MACOS) +# ============================================================================= +# On macOS with vz, network is handled automatically via NAT: +# - VMs get IP addresses from 192.168.64.0/24 via DHCP +# - No TAP devices, bridges, or iptables needed +# - The following settings are IGNORED on macOS: +# network.bridge_name, subnet_cidr, subnet_gateway, uplink_interface +network: + dns_server: 8.8.8.8 + +# ============================================================================= +# Logging +# ============================================================================= +logging: + level: debug + +# ============================================================================= +# Caddy / Ingress Configuration +# ============================================================================= +caddy: + listen_address: 0.0.0.0 + admin_address: 127.0.0.1 + admin_port: 2019 + # Note: 5353 is used by mDNSResponder (Bonjour) on macOS, using 5354 instead + internal_dns_port: 5354 + stop_on_shutdown: false + +# ============================================================================= +# Build System Configuration +# ============================================================================= +# For builds on macOS with vz, the registry URL needs to be accessible from +# NAT VMs. Since vz uses 192.168.64.0/24 for NAT, the host is at 192.168.64.1. +# +# IMPORTANT: "host.docker.internal" does NOT work in vz VMs - that's a Docker +# Desktop-specific hostname. Use the NAT gateway IP instead. +registry: + url: 192.168.64.1:8080 + insecure: true + +build: + # builder_image: "" # empty (default) = built from Dockerfile on first run + docker_socket: /var/run/docker.sock + max_concurrent_source_builds: 2 + timeout: 600 + +# ============================================================================= +# Resource Limits (same as Linux) +# ============================================================================= +limits: + max_vcpus_per_instance: 4 + max_memory_per_instance: 8GB + # max_total_volume_storage: "" # 0 or empty = unlimited + +# ============================================================================= +# OpenTelemetry (optional, same as Linux) +# ============================================================================= +# otel: +# enabled: false +# endpoint: 127.0.0.1:4317 +# service_name: hypeman +# insecure: true +# env: dev + +# ============================================================================= +# TLS / ACME Configuration (same as Linux) +# ============================================================================= +# acme: +# email: admin@example.com +# dns_provider: cloudflare +# allowed_domains: "*.example.com" +# cloudflare_api_token: "" + +# ============================================================================= +# macOS Limitations +# ============================================================================= +# The following features are NOT AVAILABLE on macOS: +# +# 1. GPU Passthrough (VFIO, mdev) +# - gpu.profile_cache_ttl is ignored +# - Device registration/binding will fail +# +# 2. Network Rate Limiting +# - network.upload_burst_multiplier, download_burst_multiplier are ignored +# - No tc/HTB equivalent on macOS +# +# 3. CPU/Memory Hotplug +# - Resize operations not supported +# +# 4. Disk I/O Limiting +# - capacity.disk_io, oversubscription.disk_io are ignored +# +# 5. Snapshots (requires macOS 14+ on Apple Silicon) +# - SaveMachineStateToPath/RestoreMachineStateFromURL require macOS 14+ +# - Only supported on ARM64 (Apple Silicon) Macs diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 00000000..de786100 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,104 @@ +# ============================================================================= +# Hypeman Server Configuration (Linux) +# ============================================================================= +# Copy this file to /etc/hypeman/config.yaml (or ~/.config/hypeman/config.yaml) +# and customize for your environment. +# +# Configuration precedence (highest to lowest): +# 1. Environment variables (e.g., JWT_SECRET, PORT) +# 2. This YAML config file +# 3. Built-in defaults +# ============================================================================= + +# Required - used to sign and verify API tokens +jwt_secret: "" + +# Data directory (default: /var/lib/hypeman) +data_dir: /var/lib/hypeman + +# Server configuration +# port: 8080 + +# ============================================================================= +# Network Configuration +# ============================================================================= +# network: +# bridge_name: vmbr0 +# subnet_cidr: 10.100.0.0/16 +# subnet_gateway: "" # empty = derived from subnet_cidr +# uplink_interface: "" # empty = auto-detect from default route +# dns_server: 1.1.1.1 + +# ============================================================================= +# Logging +# ============================================================================= +# logging: +# level: info # debug, info, warn, error + +# ============================================================================= +# Caddy / Ingress Configuration +# ============================================================================= +# caddy: +# listen_address: 0.0.0.0 +# admin_address: 127.0.0.1 +# admin_port: 0 # 0 = random (for dev); install script sets to 2019 for production +# internal_dns_port: 0 # 0 = random (for dev); install script sets to 5353 for production +# stop_on_shutdown: false # Set to true if you want Caddy to stop when hypeman stops + +# ============================================================================= +# TLS / ACME Configuration (for HTTPS ingresses) +# ============================================================================= +# Required for TLS ingresses: +# acme: +# email: admin@example.com +# dns_provider: cloudflare +# +# IMPORTANT: You must specify which domains are allowed for TLS certificates. +# This prevents typos and ensures you only request certificates for domains you control. +# allowed_domains: "*.example.com,api.other.com" +# Supports: +# - Exact matches: api.example.com +# - Wildcard subdomains: *.example.com (matches foo.example.com, NOT foo.bar.example.com) +# If not set, no TLS ingresses are allowed. +# +# Optional ACME settings: +# ca: "" # empty = Let's Encrypt production +# # Use https://acme-staging-v02.api.letsencrypt.org/directory for testing +# +# DNS propagation settings (applies to all providers): +# dns_propagation_timeout: 2m # Max time to wait for DNS propagation +# dns_resolvers: "1.1.1.1,8.8.8.8" # Custom DNS resolvers for propagation checking +# +# Cloudflare DNS Provider (dns_provider: cloudflare) +# cloudflare_api_token: your-api-token +# Token needs Zone:DNS:Edit permissions for the domains you want certificates for + +# ============================================================================= +# OpenTelemetry Configuration +# ============================================================================= +# otel: +# enabled: false +# endpoint: 127.0.0.1:4317 +# service_name: hypeman +# service_instance_id: "" # default: hostname +# insecure: true +# env: dev # deployment environment + +# ============================================================================= +# Build Configuration +# ============================================================================= +# build: +# builder_image: "" # empty = built from Dockerfile on first run +# docker_socket: /var/run/docker.sock +# max_concurrent_source_builds: 2 +# timeout: 600 + +# ============================================================================= +# Resource Limits +# ============================================================================= +# limits: +# max_vcpus_per_instance: 16 +# max_memory_per_instance: 32GB +# max_total_volume_storage: "" # 0 or empty = unlimited +# max_concurrent_builds: 1 +# max_overlay_size: 100GB diff --git a/go.mod b/go.mod index f6ce5e86..bf2cbdd1 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.4 require ( al.essio.dev/pkg/shellescape v1.6.0 + github.com/Code-Hex/vz/v3 v3.7.1 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/creack/pty v1.1.24 github.com/cyphar/filepath-securejoin v0.6.1 @@ -20,6 +21,11 @@ require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 github.com/joho/godotenv v1.5.1 + github.com/knadh/koanf/parsers/yaml v1.1.0 + github.com/knadh/koanf/providers/env v1.1.0 + github.com/knadh/koanf/providers/file v1.2.1 + github.com/knadh/koanf/providers/structs v1.0.0 + github.com/knadh/koanf/v2 v2.3.2 github.com/mdlayher/vsock v1.2.1 github.com/miekg/dns v1.1.68 github.com/nrednav/cuid2 v1.1.0 @@ -57,7 +63,6 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect - github.com/Code-Hex/vz/v3 v3.7.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect @@ -75,19 +80,25 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-test/deep v1.1.1 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mdlayher/socket v0.5.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect @@ -112,6 +123,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect go.opentelemetry.io/otel/log v0.14.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.43.0 // indirect golang.org/x/mod v0.28.0 // indirect golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 // indirect diff --git a/go.sum b/go.sum index d33f256d..7e44f095 100644 --- a/go.sum +++ b/go.sum @@ -66,9 +66,13 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -87,6 +91,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -124,6 +130,18 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4= +github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= +github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc= +github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY= +github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= +github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= +github.com/knadh/koanf/providers/structs v1.0.0 h1:DznjB7NQykhqCar2LvNug3MuxEQsZ5KvfgMbio+23u4= +github.com/knadh/koanf/providers/structs v1.0.0/go.mod h1:kjo5TFtgpaZORlpoJqcbeLowM2cINodv8kX+oFAeQ1w= +github.com/knadh/koanf/v2 v2.3.2 h1:Ee6tuzQYFwcZXQpc2MiVeC6qHMandf5SMUJJNoFp/c4= +github.com/knadh/koanf/v2 v2.3.2/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -145,8 +163,12 @@ github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnE github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= @@ -277,6 +299,8 @@ go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOV go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/integration/systemd_test.go b/integration/systemd_test.go index 79973d7c..7e0b1dff 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -48,10 +48,12 @@ func TestSystemdMode(t *testing.T) { p := paths.New(tmpDir) cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } // Create managers diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 8c02eabc..62d4f09f 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -57,10 +57,12 @@ func TestVGPU(t *testing.T) { p := paths.New(tmpDir) cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } // Create managers diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 77252ef4..416ffc3a 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -195,7 +195,7 @@ func (m *manager) Start(ctx context.Context) error { func (m *manager) ensureBuilderImage(ctx context.Context) { defer m.builderReady.Store(true) - if m.config.BuilderImage != "" && m.config.BuilderImage != "none" { + if m.config.BuilderImage != "" { // Explicit builder image configured - check if already available if _, err := m.imageManager.GetImage(ctx, m.config.BuilderImage); err == nil { m.logger.Info("builder image already available", "image", m.config.BuilderImage) diff --git a/lib/devices/gpu_e2e_test.go b/lib/devices/gpu_e2e_test.go index 848d14db..86708446 100644 --- a/lib/devices/gpu_e2e_test.go +++ b/lib/devices/gpu_e2e_test.go @@ -60,10 +60,12 @@ func TestGPUPassthrough(t *testing.T) { p := paths.New(tmpDir) cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } // Initialize managers (nil meter/tracer disables metrics/tracing) diff --git a/lib/devices/gpu_inference_test.go b/lib/devices/gpu_inference_test.go index 895c8812..c99d1b39 100644 --- a/lib/devices/gpu_inference_test.go +++ b/lib/devices/gpu_inference_test.go @@ -97,10 +97,12 @@ func TestGPUInference(t *testing.T) { p := paths.New(persistentTestDataDir) cfg := &config.Config{ - DataDir: persistentTestDataDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: persistentTestDataDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } // Initialize managers diff --git a/lib/devices/gpu_module_test.go b/lib/devices/gpu_module_test.go index 93f8d954..cd9f1b76 100644 --- a/lib/devices/gpu_module_test.go +++ b/lib/devices/gpu_module_test.go @@ -63,10 +63,12 @@ func TestNVIDIAModuleLoading(t *testing.T) { p := paths.New(tmpDir) cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } // Initialize managers @@ -308,10 +310,12 @@ func TestNVMLDetection(t *testing.T) { p := paths.New(persistentTestDataDir) cfg := &config.Config{ - DataDir: persistentTestDataDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: persistentTestDataDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } imageMgr, err := images.NewManager(p, 1, nil) diff --git a/lib/instances/manager_darwin_test.go b/lib/instances/manager_darwin_test.go index 9fde885a..95309751 100644 --- a/lib/instances/manager_darwin_test.go +++ b/lib/instances/manager_darwin_test.go @@ -37,10 +37,12 @@ func setupVZTestManager(t *testing.T) (*manager, string) { t.Cleanup(func() { os.RemoveAll(tmpDir) }) cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } p := paths.New(tmpDir) diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 95c253bd..fbc27fef 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -37,10 +37,12 @@ func setupTestManager(t *testing.T) (*manager, string) { tmpDir := t.TempDir() cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } p := paths.New(tmpDir) @@ -243,10 +245,12 @@ func TestBasicEndToEnd(t *testing.T) { // Initialize network for ingress testing networkManager := network.NewManager(p, &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, }, nil) t.Log("Initializing network...") err = networkManager.Initialize(ctx, nil) @@ -1021,10 +1025,12 @@ func TestEntrypointEnvVars(t *testing.T) { // Initialize network (needed for loopback interface in guest) networkManager := network.NewManager(p, &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, }, nil) t.Log("Initializing network...") err = networkManager.Initialize(ctx, nil) @@ -1139,14 +1145,15 @@ func TestStorageOperations(t *testing.T) { tmpDir := t.TempDir() cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(tmpDir) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index f2ffef4f..05f1f9fd 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -35,10 +35,12 @@ func setupTestManagerForQEMU(t *testing.T) (*manager, string) { tmpDir := t.TempDir() cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, } p := paths.New(tmpDir) @@ -236,10 +238,12 @@ func TestQEMUBasicEndToEnd(t *testing.T) { // Initialize network networkManager := network.NewManager(p, &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, }, nil) t.Log("Initializing network...") err = networkManager.Initialize(ctx, nil) @@ -624,10 +628,12 @@ func TestQEMUEntrypointEnvVars(t *testing.T) { // Initialize network (needed for loopback interface in guest) networkManager := network.NewManager(p, &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + Network: config.NetworkConfig{ + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + }, }, nil) t.Log("Initializing network...") err = networkManager.Initialize(ctx, nil) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index 5b50f1af..7337df8a 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -151,11 +151,10 @@ func createTestManager(t *testing.T, limits ResourceLimits) *manager { t.Helper() tmpDir := t.TempDir() cfg := &config.Config{ - DataDir: tmpDir, - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: tmpDir, + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) diff --git a/lib/network/allocate.go b/lib/network/allocate.go index aaf36f82..be0be892 100644 --- a/lib/network/allocate.go +++ b/lib/network/allocate.go @@ -81,7 +81,7 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N MAC: mac, Gateway: network.Gateway, Netmask: netmask, - DNS: m.config.DNSServer, + DNS: m.config.Network.DNSServer, TAPDevice: tap, }, nil } diff --git a/lib/network/bridge_linux.go b/lib/network/bridge_linux.go index 952d7dbb..c3cf3304 100644 --- a/lib/network/bridge_linux.go +++ b/lib/network/bridge_linux.go @@ -52,7 +52,7 @@ func (m *manager) checkSubnetConflicts(ctx context.Context, subnet string) error } // Skip if this is our own bridge (already configured from previous run) - if ifaceName == m.config.BridgeName { + if ifaceName == m.config.Network.BridgeName { continue } @@ -185,8 +185,8 @@ const ( // Uses explicit config if set, otherwise auto-detects from default route. func (m *manager) getUplinkInterface() (string, error) { // Explicit config takes precedence - if m.config.UplinkInterface != "" { - return m.config.UplinkInterface, nil + if m.config.Network.UplinkInterface != "" { + return m.config.Network.UplinkInterface, nil } // Auto-detect from default route @@ -722,7 +722,7 @@ func formatTcRate(bytesPerSec int64) string { // deleteTAPDevice removes TAP device and its associated HTB class on the bridge. func (m *manager) deleteTAPDevice(tapName string) error { // Remove HTB class from bridge before deleting TAP - m.removeVMClass(m.config.BridgeName, tapName) + m.removeVMClass(m.config.Network.BridgeName, tapName) link, err := netlink.LinkByName(tapName) if err != nil { @@ -832,7 +832,7 @@ func (m *manager) CleanupOrphanedTAPs(ctx context.Context, runningInstanceIDs [] // Returns the number of classes deleted. func (m *manager) CleanupOrphanedClasses(ctx context.Context) int { log := logger.FromContext(ctx) - bridgeName := m.config.BridgeName + bridgeName := m.config.Network.BridgeName // List all HTB classes on the bridge cmd := exec.Command("tc", "class", "show", "dev", bridgeName) diff --git a/lib/network/manager.go b/lib/network/manager.go index 5fed5d11..cf73002d 100644 --- a/lib/network/manager.go +++ b/lib/network/manager.go @@ -71,28 +71,28 @@ func (m *manager) Initialize(ctx context.Context, runningInstanceIDs []string) e log := logger.FromContext(ctx) // Derive gateway from subnet if not explicitly configured - gateway := m.config.SubnetGateway + gateway := m.config.Network.SubnetGateway if gateway == "" { var err error - gateway, err = DeriveGateway(m.config.SubnetCIDR) + gateway, err = DeriveGateway(m.config.Network.SubnetCIDR) if err != nil { return fmt.Errorf("derive gateway from subnet: %w", err) } } log.InfoContext(ctx, "initializing network manager", - "bridge", m.config.BridgeName, - "subnet", m.config.SubnetCIDR, + "bridge", m.config.Network.BridgeName, + "subnet", m.config.Network.SubnetCIDR, "gateway", gateway) // Check for subnet conflicts with existing host routes before creating bridge - if err := m.checkSubnetConflicts(ctx, m.config.SubnetCIDR); err != nil { + if err := m.checkSubnetConflicts(ctx, m.config.Network.SubnetCIDR); err != nil { return err } // Ensure default network bridge exists and iptables rules are configured // createBridge is idempotent - handles both new and existing bridges - if err := m.createBridge(ctx, m.config.BridgeName, gateway, m.config.SubnetCIDR); err != nil { + if err := m.createBridge(ctx, m.config.Network.BridgeName, gateway, m.config.Network.SubnetCIDR); err != nil { return fmt.Errorf("setup default network: %w", err) } @@ -113,7 +113,7 @@ func (m *manager) Initialize(ctx context.Context, runningInstanceIDs []string) e // getDefaultNetwork gets the default network details from kernel state func (m *manager) getDefaultNetwork(ctx context.Context) (*Network, error) { // Query from kernel - state, err := m.queryNetworkState(m.config.BridgeName) + state, err := m.queryNetworkState(m.config.Network.BridgeName) if err != nil { return nil, ErrNotFound } @@ -122,7 +122,7 @@ func (m *manager) getDefaultNetwork(ctx context.Context) (*Network, error) { Name: "default", Subnet: state.Subnet, Gateway: state.Gateway, - Bridge: m.config.BridgeName, + Bridge: m.config.Network.BridgeName, Isolated: true, Default: true, CreatedAt: time.Time{}, // Unknown for default @@ -132,23 +132,23 @@ func (m *manager) getDefaultNetwork(ctx context.Context) (*Network, error) { // SetupHTB initializes HTB qdisc on the bridge for upload fair sharing. // capacityBps is the total network capacity in bytes per second. func (m *manager) SetupHTB(ctx context.Context, capacityBps int64) error { - return m.setupBridgeHTB(ctx, m.config.BridgeName, capacityBps) + return m.setupBridgeHTB(ctx, m.config.Network.BridgeName, capacityBps) } // GetUploadBurstMultiplier returns the configured multiplier for upload burst ceiling. // Defaults to 4 if not configured. func (m *manager) GetUploadBurstMultiplier() int { - if m.config.UploadBurstMultiplier < 1 { + if m.config.Network.UploadBurstMultiplier < 1 { return DefaultUploadBurstMultiplier } - return m.config.UploadBurstMultiplier + return m.config.Network.UploadBurstMultiplier } // GetDownloadBurstMultiplier returns the configured multiplier for download burst bucket. // Defaults to 4 if not configured. func (m *manager) GetDownloadBurstMultiplier() int { - if m.config.DownloadBurstMultiplier < 1 { + if m.config.Network.DownloadBurstMultiplier < 1 { return DefaultDownloadBurstMultiplier } - return m.config.DownloadBurstMultiplier + return m.config.Network.DownloadBurstMultiplier } diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 1963fcb5..221388e6 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -53,8 +53,13 @@ func ProvideContext(log *slog.Logger) context.Context { // ProvideConfig provides the application configuration. // Panics if configuration is invalid (prevents startup with bad config). +// Config path can be specified via CONFIG_PATH env var or defaults to platform-specific locations. func ProvideConfig() *config.Config { - cfg := config.Load() + configPath := os.Getenv("CONFIG_PATH") + cfg, err := config.Load(configPath) + if err != nil { + panic(fmt.Sprintf("failed to load configuration: %v", err)) + } if err := cfg.Validate(); err != nil { panic(fmt.Sprintf("invalid configuration: %v", err)) } @@ -69,7 +74,7 @@ func ProvidePaths(cfg *config.Config) *paths.Paths { // ProvideImageManager provides the image manager func ProvideImageManager(p *paths.Paths, cfg *config.Config) (images.Manager, error) { meter := otel.GetMeterProvider().Meter("hypeman") - return images.NewManager(p, cfg.MaxConcurrentBuilds, meter) + return images.NewManager(p, cfg.Limits.MaxConcurrentBuilds, meter) } // ProvideSystemManager provides the system manager @@ -92,16 +97,16 @@ func ProvideDeviceManager(p *paths.Paths) devices.Manager { func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager) (instances.Manager, error) { // Parse max overlay size from config var maxOverlaySize datasize.ByteSize - if err := maxOverlaySize.UnmarshalText([]byte(cfg.MaxOverlaySize)); err != nil { - return nil, fmt.Errorf("failed to parse MAX_OVERLAY_SIZE '%s': %w (expected format like '100GB', '50G', '10GiB')", cfg.MaxOverlaySize, err) + if err := maxOverlaySize.UnmarshalText([]byte(cfg.Limits.MaxOverlaySize)); err != nil { + return nil, fmt.Errorf("failed to parse MAX_OVERLAY_SIZE '%s': %w (expected format like '100GB', '50G', '10GiB')", cfg.Limits.MaxOverlaySize, err) } // Parse max memory per instance (empty or "0" means unlimited) var maxMemoryPerInstance int64 - if cfg.MaxMemoryPerInstance != "" && cfg.MaxMemoryPerInstance != "0" { + if cfg.Limits.MaxMemoryPerInstance != "" && cfg.Limits.MaxMemoryPerInstance != "0" { var memSize datasize.ByteSize - if err := memSize.UnmarshalText([]byte(cfg.MaxMemoryPerInstance)); err != nil { - return nil, fmt.Errorf("failed to parse MAX_MEMORY_PER_INSTANCE '%s': %w", cfg.MaxMemoryPerInstance, err) + if err := memSize.UnmarshalText([]byte(cfg.Limits.MaxMemoryPerInstance)); err != nil { + return nil, fmt.Errorf("failed to parse MAX_MEMORY_PER_INSTANCE '%s': %w", cfg.Limits.MaxMemoryPerInstance, err) } maxMemoryPerInstance = int64(memSize) } @@ -110,13 +115,13 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima // in the ResourceManager, wired up via SetResourceValidator after initialization. limits := instances.ResourceLimits{ MaxOverlaySize: int64(maxOverlaySize), - MaxVcpusPerInstance: cfg.MaxVcpusPerInstance, + MaxVcpusPerInstance: cfg.Limits.MaxVcpusPerInstance, MaxMemoryPerInstance: maxMemoryPerInstance, } meter := otel.GetMeterProvider().Meter("hypeman") tracer := otel.GetTracerProvider().Tracer("hypeman") - defaultHypervisor := hypervisor.Type(cfg.DefaultHypervisor) + defaultHypervisor := hypervisor.Type(cfg.Hypervisor.Default) return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, meter, tracer), nil } @@ -124,10 +129,10 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima func ProvideVolumeManager(p *paths.Paths, cfg *config.Config) (volumes.Manager, error) { // Parse max total volume storage (empty or "0" means unlimited) var maxTotalVolumeStorage int64 - if cfg.MaxTotalVolumeStorage != "" && cfg.MaxTotalVolumeStorage != "0" { + if cfg.Limits.MaxTotalVolumeStorage != "" && cfg.Limits.MaxTotalVolumeStorage != "0" { var storageSize datasize.ByteSize - if err := storageSize.UnmarshalText([]byte(cfg.MaxTotalVolumeStorage)); err != nil { - return nil, fmt.Errorf("failed to parse MAX_TOTAL_VOLUME_STORAGE '%s': %w", cfg.MaxTotalVolumeStorage, err) + if err := storageSize.UnmarshalText([]byte(cfg.Limits.MaxTotalVolumeStorage)); err != nil { + return nil, fmt.Errorf("failed to parse MAX_TOTAL_VOLUME_STORAGE '%s': %w", cfg.Limits.MaxTotalVolumeStorage, err) } maxTotalVolumeStorage = int64(storageSize) } @@ -178,20 +183,20 @@ func ProvideVMMetricsManager(instanceManager instances.Manager) (*vm_metrics.Man // ProvideIngressManager provides the ingress manager func ProvideIngressManager(p *paths.Paths, cfg *config.Config, instanceManager instances.Manager) (ingress.Manager, error) { // Parse DNS provider - fail if invalid - dnsProvider, err := ingress.ParseDNSProvider(cfg.AcmeDnsProvider) + dnsProvider, err := ingress.ParseDNSProvider(cfg.ACME.DNSProvider) if err != nil { return nil, fmt.Errorf("invalid ACME_DNS_PROVIDER: %w", err) } // Validate DNS propagation timeout if set (must be a valid Go duration string) - if cfg.DnsPropagationTimeout != "" { - if _, err := time.ParseDuration(cfg.DnsPropagationTimeout); err != nil { - return nil, fmt.Errorf("invalid DNS_PROPAGATION_TIMEOUT %q: %w (expected format like '2m', '120s', '1h')", cfg.DnsPropagationTimeout, err) + if cfg.ACME.DNSPropagationTimeout != "" { + if _, err := time.ParseDuration(cfg.ACME.DNSPropagationTimeout); err != nil { + return nil, fmt.Errorf("invalid DNS_PROPAGATION_TIMEOUT %q: %w (expected format like '2m', '120s', '1h')", cfg.ACME.DNSPropagationTimeout, err) } } // Use config value for internal DNS port, fall back to default (0 = random) if not set - internalDNSPort := cfg.InternalDNSPort + internalDNSPort := cfg.Caddy.InternalDNSPort if internalDNSPort == 0 { internalDNSPort = ingress.DefaultDNSPort } @@ -205,25 +210,25 @@ func ProvideIngressManager(p *paths.Paths, cfg *config.Config, instanceManager i } ingressConfig := ingress.Config{ - ListenAddress: cfg.CaddyListenAddress, - AdminAddress: cfg.CaddyAdminAddress, - AdminPort: cfg.CaddyAdminPort, + ListenAddress: cfg.Caddy.ListenAddress, + AdminAddress: cfg.Caddy.AdminAddress, + AdminPort: cfg.Caddy.AdminPort, DNSPort: internalDNSPort, - StopOnShutdown: cfg.CaddyStopOnShutdown, + StopOnShutdown: cfg.Caddy.StopOnShutdown, ACME: ingress.ACMEConfig{ - Email: cfg.AcmeEmail, + Email: cfg.ACME.Email, DNSProvider: dnsProvider, - CA: cfg.AcmeCA, - DNSPropagationTimeout: cfg.DnsPropagationTimeout, - DNSResolvers: cfg.DnsResolvers, - AllowedDomains: cfg.TlsAllowedDomains, - CloudflareAPIToken: cfg.CloudflareApiToken, + CA: cfg.ACME.CA, + DNSPropagationTimeout: cfg.ACME.DNSPropagationTimeout, + DNSResolvers: cfg.ACME.DNSResolvers, + AllowedDomains: cfg.ACME.AllowedDomains, + CloudflareAPIToken: cfg.ACME.CloudflareAPIToken, }, APIIngress: ingress.APIIngressConfig{ - Hostname: cfg.ApiHostname, + Hostname: cfg.API.Hostname, Port: apiPort, - TLS: cfg.ApiTLS, - RedirectHTTP: cfg.ApiRedirectHTTP, + TLS: cfg.API.TLS, + RedirectHTTP: cfg.API.RedirectHTTP, }, } @@ -243,62 +248,51 @@ func ProvideIngressManager(p *paths.Paths, cfg *config.Config, instanceManager i func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager instances.Manager, volumeManager volumes.Manager, imageManager images.Manager, log *slog.Logger) (builds.Manager, error) { // Read CA cert file if specified var registryCACert string - if cfg.RegistryCACertFile != "" { - certData, err := os.ReadFile(cfg.RegistryCACertFile) + if cfg.Registry.CACertFile != "" { + certData, err := os.ReadFile(cfg.Registry.CACertFile) if err != nil { return nil, fmt.Errorf("read registry CA cert file: %w", err) } registryCACert = string(certData) - log.Info("registry CA certificate loaded", "file", cfg.RegistryCACertFile) + log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } // Rewrite localhost in RegistryURL to the subnet gateway IP so builder VMs // (which run in their own network namespace) can reach the host registry. // Inside a VM, "localhost" refers to the VM itself, not the host. - registryURL := cfg.RegistryURL + registryURL := cfg.Registry.URL if registryURL == "" { registryURL = "localhost:8080" } if strings.HasPrefix(registryURL, "localhost:") || strings.HasPrefix(registryURL, "127.0.0.1:") { - gateway := cfg.SubnetGateway + gateway := cfg.Network.SubnetGateway if gateway == "" { var err error - gateway, err = network.DeriveGateway(cfg.SubnetCIDR) + gateway, err = network.DeriveGateway(cfg.Network.SubnetCIDR) if err != nil { return nil, fmt.Errorf("derive gateway for registry URL rewrite: %w", err) } } port := strings.SplitN(registryURL, ":", 2)[1] registryURL = gateway + ":" + port - log.Info("rewrote registry URL for builder VMs", "original", cfg.RegistryURL, "rewritten", registryURL) + log.Info("rewrote registry URL for builder VMs", "original", cfg.Registry.URL, "rewritten", registryURL) } buildConfig := builds.Config{ - MaxConcurrentBuilds: cfg.MaxConcurrentSourceBuilds, - BuilderImage: cfg.BuilderImage, + MaxConcurrentBuilds: cfg.Build.MaxConcurrentSourceBuilds, + BuilderImage: cfg.Build.BuilderImage, RegistryURL: registryURL, - RegistryInsecure: cfg.RegistryInsecure, + RegistryInsecure: cfg.Registry.Insecure, RegistryCACert: registryCACert, - DefaultTimeout: cfg.BuildTimeout, + DefaultTimeout: cfg.Build.Timeout, RegistrySecret: cfg.JwtSecret, // Use same secret for registry tokens } - // Apply defaults if not set - if buildConfig.MaxConcurrentBuilds == 0 { - buildConfig.MaxConcurrentBuilds = 2 - } - if buildConfig.BuilderImage == "" { - buildConfig.BuilderImage = "hypeman/builder:latest" - } - if buildConfig.DefaultTimeout == 0 { - buildConfig.DefaultTimeout = 600 - } - // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics) var secretProvider builds.SecretProvider - if cfg.BuildSecretsDir != "" { - secretProvider = builds.NewFileSecretProvider(cfg.BuildSecretsDir) - log.Info("build secrets enabled", "dir", cfg.BuildSecretsDir) + if cfg.Build.SecretsDir != "" { + secretProvider = builds.NewFileSecretProvider(cfg.Build.SecretsDir) + log.Info("build secrets enabled", "dir", cfg.Build.SecretsDir) } else { secretProvider = &builds.NoOpSecretProvider{} } diff --git a/lib/resources/disk.go b/lib/resources/disk.go index d431ec20..c8847ce7 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk.go @@ -20,14 +20,14 @@ type DiskResource struct { } // NewDiskResource discovers disk capacity for the data directory. -// If cfg.DiskLimit is set, uses that as capacity; otherwise auto-detects via statfs. +// If cfg.Capacity.Disk is set, uses that as capacity; otherwise auto-detects via statfs. func NewDiskResource(cfg *config.Config, p *paths.Paths, instLister InstanceLister, imgLister ImageLister, volLister VolumeLister) (*DiskResource, error) { var capacity int64 - if cfg.DiskLimit != "" { + if cfg.Capacity.Disk != "" { // Parse configured limit var ds datasize.ByteSize - if err := ds.UnmarshalText([]byte(cfg.DiskLimit)); err != nil { + if err := ds.UnmarshalText([]byte(cfg.Capacity.Disk)); err != nil { return nil, err } capacity = int64(ds.Bytes()) diff --git a/lib/resources/network_linux.go b/lib/resources/network_linux.go index 6fa285f1..0ec292a0 100644 --- a/lib/resources/network_linux.go +++ b/lib/resources/network_linux.go @@ -21,21 +21,21 @@ type NetworkResource struct { } // NewNetworkResource discovers network capacity. -// If cfg.NetworkLimit is set, uses that; otherwise auto-detects from uplink interface. +// If cfg.Capacity.Network is set, uses that; otherwise auto-detects from uplink interface. func NewNetworkResource(ctx context.Context, cfg *config.Config, instLister InstanceLister) (*NetworkResource, error) { var capacity int64 log := logger.FromContext(ctx) - if cfg.NetworkLimit != "" { + if cfg.Capacity.Network != "" { // Parse configured limit (e.g., "10Gbps", "1GB/s") - parsed, err := ParseBandwidth(cfg.NetworkLimit) + parsed, err := ParseBandwidth(cfg.Capacity.Network) if err != nil { return nil, fmt.Errorf("parse network limit: %w", err) } capacity = parsed } else { // Auto-detect from uplink interface - uplink, err := getUplinkInterface(cfg.UplinkInterface) + uplink, err := getUplinkInterface(cfg.Network.UplinkInterface) if err != nil { // No uplink found - network limiting disabled log.WarnContext(ctx, "no uplink interface found, network limiting disabled", "error", err) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 939fdcbb..b22f7978 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -228,15 +228,15 @@ func (m *Manager) GetOversubRatio(rt ResourceType) float64 { var ratio float64 switch rt { case ResourceCPU: - ratio = m.cfg.OversubCPU + ratio = m.cfg.Oversubscription.CPU case ResourceMemory: - ratio = m.cfg.OversubMemory + ratio = m.cfg.Oversubscription.Memory case ResourceDisk: - ratio = m.cfg.OversubDisk + ratio = m.cfg.Oversubscription.Disk case ResourceNetwork: - ratio = m.cfg.OversubNetwork + ratio = m.cfg.Oversubscription.Network case ResourceDiskIO: - ratio = m.cfg.OversubDiskIO + ratio = m.cfg.Oversubscription.DiskIO default: return 1.0 } @@ -282,7 +282,7 @@ func (m *Manager) GetStatus(ctx context.Context, rt ResourceType) (*ResourceStat // Add source info for network if rt == ResourceNetwork { - if m.cfg.NetworkLimit != "" { + if m.cfg.Capacity.Network != "" { status.Source = SourceConfigured } else { status.Source = SourceDetected @@ -483,11 +483,11 @@ func (m *Manager) NetworkCapacity() int64 { // DiskIOCapacity returns the disk I/O capacity in bytes/sec. // Uses configured DISK_IO_LIMIT if set, otherwise defaults to 1 GB/s. func (m *Manager) DiskIOCapacity() int64 { - if m.cfg.DiskIOLimit == "" { + if m.cfg.Capacity.DiskIO == "" { return 1 * 1000 * 1000 * 1000 // 1 GB/s default } // Parse the limit using the same format as network (e.g., "500MB/s") - capacity, err := parseDiskIOLimit(m.cfg.DiskIOLimit) + capacity, err := parseDiskIOLimit(m.cfg.Capacity.DiskIO) if err != nil { return 1 * 1000 * 1000 * 1000 // 1 GB/s fallback } @@ -534,7 +534,7 @@ func (m *Manager) DefaultDiskIOBandwidth(vcpus int) (ioBps, burstBps int64) { return 0, 0 } - ratio := m.cfg.OversubDiskIO + ratio := m.cfg.Oversubscription.DiskIO if ratio <= 0 { ratio = 2.0 // Default 2x oversubscription for disk I/O } @@ -579,7 +579,7 @@ func (m *Manager) MaxImageStorageBytes() int64 { } capacity := diskRes.Capacity() - fraction := m.cfg.MaxImageStorage + fraction := m.cfg.Limits.MaxImageStorage if fraction <= 0 { fraction = 0.2 // Default 20% } @@ -617,7 +617,7 @@ func (m *Manager) HasSufficientImageStorage(ctx context.Context) error { max := m.MaxImageStorageBytes() if max > 0 && current >= max { return fmt.Errorf("image storage limit exceeded: %d bytes used, limit is %d bytes (%.0f%% of disk)", - current, max, m.cfg.MaxImageStorage*100) + current, max, m.cfg.Limits.MaxImageStorage*100) } return nil diff --git a/lib/resources/resource_test.go b/lib/resources/resource_test.go index 3fb9d66a..0480561f 100644 --- a/lib/resources/resource_test.go +++ b/lib/resources/resource_test.go @@ -45,11 +45,10 @@ func (m *mockVolumeLister) TotalVolumeBytes(ctx context.Context) (int64, error) func TestNewManager(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 2.0, - OversubMemory: 1.5, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 2.0, Memory: 1.5, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) @@ -59,11 +58,10 @@ func TestNewManager(t *testing.T) { func TestGetOversubRatio(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 2.0, - OversubMemory: 1.5, - OversubDisk: 1.0, - OversubNetwork: 3.0, + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 2.0, Memory: 1.5, Disk: 1.0, Network: 3.0, + }, } p := paths.New(cfg.DataDir) @@ -78,12 +76,11 @@ func TestGetOversubRatio(t *testing.T) { func TestDefaultNetworkBandwidth(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, - NetworkLimit: "10Gbps", // 1.25 GB/s = 1,250,000,000 bytes/sec + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, + Capacity: config.CapacityConfig{Network: "10Gbps"}, // 1.25 GB/s = 1,250,000,000 bytes/sec } p := paths.New(cfg.DataDir) @@ -111,11 +108,10 @@ func TestDefaultNetworkBandwidth(t *testing.T) { func TestDefaultNetworkBandwidth_ZeroCPU(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) @@ -224,11 +220,10 @@ func TestIsActiveState(t *testing.T) { func TestHasSufficientDiskForPull(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) @@ -252,11 +247,10 @@ func TestHasSufficientDiskForPull(t *testing.T) { // This catches a bug where CPU/Memory SetInstanceLister was not being called. func TestInitialize_SetsInstanceListersForAllResources(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) @@ -301,12 +295,11 @@ func TestInitialize_SetsInstanceListersForAllResources(t *testing.T) { // returns correct allocations for all resource types. func TestGetFullStatus_ReturnsAllResourceAllocations(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 2.0, - OversubMemory: 1.5, - OversubDisk: 1.0, - OversubNetwork: 1.0, - NetworkLimit: "10Gbps", + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 2.0, Memory: 1.5, Disk: 1.0, Network: 1.0, + }, + Capacity: config.CapacityConfig{Network: "10Gbps"}, } p := paths.New(cfg.DataDir) @@ -358,9 +351,9 @@ func TestNetworkResource_Allocated(t *testing.T) { t.Skip("network rate limiting not supported on this platform") } cfg := &config.Config{ - DataDir: t.TempDir(), - NetworkLimit: "1Gbps", // 125MB/s - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Capacity: config.CapacityConfig{Network: "1Gbps"}, // 125MB/s + Oversubscription: config.OversubscriptionConfig{Network: 1.0}, } mockLister := &mockInstanceLister{ @@ -383,12 +376,11 @@ func TestNetworkResource_Allocated(t *testing.T) { // TestMaxImageStorage verifies the image storage limit calculation func TestMaxImageStorage(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - MaxImageStorage: 0.2, // 20% - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Limits: config.LimitsConfig{MaxImageStorage: 0.2}, // 20% + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) @@ -419,11 +411,10 @@ func TestMaxImageStorage(t *testing.T) { // includes OCI cache and volume overlays func TestDiskBreakdown_IncludesOCICacheAndVolumeOverlays(t *testing.T) { cfg := &config.Config{ - DataDir: t.TempDir(), - OversubCPU: 1.0, - OversubMemory: 1.0, - OversubDisk: 1.0, - OversubNetwork: 1.0, + DataDir: t.TempDir(), + Oversubscription: config.OversubscriptionConfig{ + CPU: 1.0, Memory: 1.0, Disk: 1.0, Network: 1.0, + }, } p := paths.New(cfg.DataDir) diff --git a/scripts/build-from-source.sh b/scripts/build-from-source.sh index ecfe28c8..17489b36 100755 --- a/scripts/build-from-source.sh +++ b/scripts/build-from-source.sh @@ -80,8 +80,13 @@ if ! go build -o "${OUTPUT_DIR}/hypeman-token" ./cmd/gen-jwt >> "$BUILD_LOG" 2>& error "Failed to build hypeman-token" fi -# Copy .env.example for config template -cp ".env.example" "${OUTPUT_DIR}/.env.example" +# Copy config example files for config template +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +if [ "$OS" = "darwin" ]; then + cp "config.example.darwin.yaml" "${OUTPUT_DIR}/config.example.darwin.yaml" +else + cp "config.example.yaml" "${OUTPUT_DIR}/config.example.yaml" +fi info "Build complete" info "Binaries are available in: ${OUTPUT_DIR}" diff --git a/scripts/e2e-install-test.sh b/scripts/e2e-install-test.sh index cf7f1324..764d0353 100755 --- a/scripts/e2e-install-test.sh +++ b/scripts/e2e-install-test.sh @@ -37,7 +37,8 @@ KEEP_DATA=false bash scripts/uninstall.sh 2>/dev/null || true # ============================================================================= info "Phase 2: Installing from source..." BRANCH=$(git rev-parse --abbrev-ref HEAD) -BRANCH="$BRANCH" bash scripts/install.sh +# Build CLI from source too when CLI_BRANCH is set (e.g., for testing unreleased CLI features) +BRANCH="$BRANCH" CLI_BRANCH="${CLI_BRANCH:-}" bash scripts/install.sh # ============================================================================= # Phase 3: Wait for service @@ -101,43 +102,30 @@ else fi fi -# Check config +# Check config files if [ "$OS" = "darwin" ]; then - [ -f "$HOME/.config/hypeman/config" ] || fail "Config file not found" + [ -f "$HOME/.config/hypeman/config.yaml" ] || fail "Server config file not found" else - [ -f /etc/hypeman/config ] || fail "Config file not found" + [ -f /etc/hypeman/config.yaml ] || fail "Server config file not found" fi -pass "Config file exists" +pass "Server config file exists" + +[ -f "$HOME/.config/hypeman/cli.yaml" ] || fail "CLI config file not found" +pass "CLI config file exists" # ============================================================================= # Phase 4b: Testing CLI commands # ============================================================================= info "Phase 4b: Testing CLI commands..." -# Determine config file path -if [ "$OS" = "darwin" ]; then - CONFIG_FILE="$HOME/.config/hypeman/config" -else - CONFIG_FILE="/etc/hypeman/config" -fi - -# Extract JWT_SECRET and PORT from config (source is unsafe — values may contain spaces) -JWT_SECRET=$(grep '^JWT_SECRET=' "$CONFIG_FILE" | cut -d= -f2-) -PORT=$(grep '^PORT=' "$CONFIG_FILE" | cut -d= -f2-) -export JWT_SECRET - -# Generate API token using hypeman-token +# hypeman-token should be able to find jwt_secret from config.yaml automatically if [ "$OS" = "darwin" ]; then API_KEY=$("/usr/local/bin/hypeman-token" -user-id "e2e-test-user") else - API_KEY=$("/opt/hypeman/bin/hypeman-token" -user-id "e2e-test-user") + API_KEY=$("/usr/local/bin/hypeman-token" -user-id "e2e-test-user") fi -[ -n "$API_KEY" ] || fail "Failed to generate API token" -pass "Generated API token" - -# Set CLI env -export HYPEMAN_API_KEY="$API_KEY" -export HYPEMAN_BASE_URL="http://localhost:${PORT:-8080}" +[ -n "$API_KEY" ] || fail "Failed to generate API token (hypeman-token should find jwt_secret from config.yaml)" +pass "hypeman-token reads jwt_secret from config.yaml" # Determine CLI path HYPEMAN_CMD="/usr/local/bin/hypeman" diff --git a/scripts/install.sh b/scripts/install.sh index 367b6db5..896fa12a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -9,6 +9,7 @@ # VERSION - Install specific API version (default: latest) # CLI_VERSION - Install specific CLI version (default: latest) # BRANCH - Build from source using this branch (for development/testing) +# CLI_BRANCH - Build CLI from source using this branch of kernel/hypeman-cli # BINARY_DIR - Use binaries from this directory instead of building/downloading # INSTALL_DIR - Binary installation directory (default: /opt/hypeman/bin on Linux, /usr/local/bin on macOS) # DATA_DIR - Data directory (default: /var/lib/hypeman on Linux, ~/Library/Application Support/hypeman on macOS) @@ -101,7 +102,7 @@ else CONFIG_DIR="${CONFIG_DIR:-/etc/hypeman}" fi -CONFIG_FILE="${CONFIG_DIR}/config" +CONFIG_FILE="${CONFIG_DIR}/config.yaml" SYSTEMD_DIR="/etc/systemd/system" # ============================================================================= @@ -248,15 +249,15 @@ if [ -n "$BINARY_DIR" ]; then info "Copying binaries from ${BINARY_DIR}..." if [ "$OS" = "darwin" ]; then - for f in "${BINARY_NAME}" "hypeman-token" ".env.darwin.example"; do + for f in "${BINARY_NAME}" "hypeman-token" "config.example.darwin.yaml"; do [ -f "${BINARY_DIR}/${f}" ] || error "File ${f} not found in ${BINARY_DIR}" done - cp "${BINARY_DIR}/.env.darwin.example" "${TMP_DIR}/.env.darwin.example" + cp "${BINARY_DIR}/config.example.darwin.yaml" "${TMP_DIR}/config.example.darwin.yaml" else - for f in "${BINARY_NAME}" "hypeman-token" ".env.example"; do + for f in "${BINARY_NAME}" "hypeman-token" "config.example.yaml"; do [ -f "${BINARY_DIR}/${f}" ] || error "File ${f} not found in ${BINARY_DIR}" done - cp "${BINARY_DIR}/.env.example" "${TMP_DIR}/.env.example" + cp "${BINARY_DIR}/config.example.yaml" "${TMP_DIR}/config.example.yaml" fi cp "${BINARY_DIR}/${BINARY_NAME}" "${TMP_DIR}/${BINARY_NAME}" @@ -295,9 +296,9 @@ elif [ -n "$BRANCH" ]; then cat "$BUILD_LOG" error "Signing failed" fi - cp ".env.darwin.example" "${TMP_DIR}/.env.darwin.example" + cp "config.example.darwin.yaml" "${TMP_DIR}/config.example.darwin.yaml" else - cp ".env.example" "${TMP_DIR}/.env.example" + cp "config.example.yaml" "${TMP_DIR}/config.example.yaml" fi cp "bin/hypeman" "${TMP_DIR}/${BINARY_NAME}" @@ -392,17 +393,9 @@ info "Installing hypeman-token to ${INSTALL_DIR}..." $SUDO install -m 755 "${TMP_DIR}/hypeman-token" "${INSTALL_DIR}/hypeman-token" if [ "$OS" = "linux" ]; then - # Install wrapper script to /usr/local/bin for easy access - info "Installing hypeman-token wrapper to /usr/local/bin..." - $SUDO tee /usr/local/bin/hypeman-token > /dev/null << EOF -#!/bin/bash -# Wrapper script for hypeman-token that loads config from ${CONFIG_FILE} -set -a -source ${CONFIG_FILE} -set +a -exec ${INSTALL_DIR}/hypeman-token "\$@" -EOF - $SUDO chmod 755 /usr/local/bin/hypeman-token + # Symlink to /usr/local/bin for easy access + info "Linking hypeman-token to /usr/local/bin..." + $SUDO ln -sf "${INSTALL_DIR}/hypeman-token" /usr/local/bin/hypeman-token fi # ============================================================================= @@ -429,26 +422,31 @@ fi # ============================================================================= if [ ! -f "$CONFIG_FILE" ]; then + info "Generating JWT secret..." + JWT_SECRET=$(openssl rand -hex 32) + if [ "$OS" = "darwin" ]; then - # macOS config - if [ -f "${TMP_DIR}/.env.darwin.example" ]; then + # macOS config - use template from source or download it + if [ -f "${TMP_DIR}/config.example.darwin.yaml" ]; then info "Using macOS config template from source..." - cp "${TMP_DIR}/.env.darwin.example" "${TMP_DIR}/config" + cp "${TMP_DIR}/config.example.darwin.yaml" "${TMP_DIR}/config.yaml" else info "Downloading macOS config template..." - CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/.env.darwin.example" - if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config"; then + CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/config.example.darwin.yaml" + if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config.yaml"; then error "Failed to download config template from ${CONFIG_URL}" fi fi - # Expand ~ to $HOME (launchd doesn't do shell expansion) - sed -i '' "s|~/|${HOME}/|g" "${TMP_DIR}/config" + # Expand ~ to $HOME in data_dir (launchd doesn't do shell expansion) + sed -i '' "s|~/|${HOME}/|g" "${TMP_DIR}/config.yaml" - # Generate random JWT secret - info "Generating JWT secret..." - JWT_SECRET=$(openssl rand -hex 32) - sed -i '' "s/^JWT_SECRET=.*/JWT_SECRET=${JWT_SECRET}/" "${TMP_DIR}/config" + # Set jwt_secret in the config + if grep -q '^jwt_secret:' "${TMP_DIR}/config.yaml"; then + sed -i '' "s|^jwt_secret:.*|jwt_secret: \"${JWT_SECRET}\"|" "${TMP_DIR}/config.yaml" + else + error "Config template missing jwt_secret field — cannot configure authentication" + fi # Auto-detect Docker socket DOCKER_SOCKET="" @@ -461,45 +459,61 @@ if [ ! -f "$CONFIG_FILE" ]; then fi if [ -n "$DOCKER_SOCKET" ]; then info "Detected Docker socket: ${DOCKER_SOCKET}" - if grep -q '^DOCKER_SOCKET=' "${TMP_DIR}/config"; then - sed -i '' "s|^DOCKER_SOCKET=.*|DOCKER_SOCKET=${DOCKER_SOCKET}|" "${TMP_DIR}/config" - elif grep -q '^# DOCKER_SOCKET=' "${TMP_DIR}/config"; then - sed -i '' "s|^# DOCKER_SOCKET=.*|DOCKER_SOCKET=${DOCKER_SOCKET}|" "${TMP_DIR}/config" + # Docker socket is nested under build: in the config + if grep -q '^build:' "${TMP_DIR}/config.yaml"; then + # build: section exists, check for docker_socket within it + if grep -q '^ docker_socket:' "${TMP_DIR}/config.yaml"; then + sed -i '' "s|^ docker_socket:.*| docker_socket: \"${DOCKER_SOCKET}\"|" "${TMP_DIR}/config.yaml" + else + # Append docker_socket after the build: line (BSD-compatible) + sed -i '' "s|^build:|build:\\ + docker_socket: \"${DOCKER_SOCKET}\"|" "${TMP_DIR}/config.yaml" + fi else - echo "DOCKER_SOCKET=${DOCKER_SOCKET}" >> "${TMP_DIR}/config" + # No build: section, add one + printf "\nbuild:\n docker_socket: \"%s\"\n" "${DOCKER_SOCKET}" >> "${TMP_DIR}/config.yaml" fi fi info "Installing config file at ${CONFIG_FILE}..." - install -m 600 "${TMP_DIR}/config" "$CONFIG_FILE" + install -m 600 "${TMP_DIR}/config.yaml" "$CONFIG_FILE" else - # Linux config - if [ -f "${TMP_DIR}/.env.example" ]; then + # Linux config - use template from source or download it + if [ -f "${TMP_DIR}/config.example.yaml" ]; then info "Using config template from source..." - cp "${TMP_DIR}/.env.example" "${TMP_DIR}/config" + cp "${TMP_DIR}/config.example.yaml" "${TMP_DIR}/config.yaml" else info "Downloading config template..." - CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/.env.example" - if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config"; then + CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/config.example.yaml" + if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config.yaml"; then error "Failed to download config template from ${CONFIG_URL}" fi fi - # Generate random JWT secret - info "Generating JWT secret..." - JWT_SECRET=$(openssl rand -hex 32) - sed -i "s/^JWT_SECRET=$/JWT_SECRET=${JWT_SECRET}/" "${TMP_DIR}/config" + # Set jwt_secret in the config + if grep -q '^jwt_secret:' "${TMP_DIR}/config.yaml"; then + sed -i "s|^jwt_secret:.*|jwt_secret: \"${JWT_SECRET}\"|" "${TMP_DIR}/config.yaml" + else + error "Config template missing jwt_secret field — cannot configure authentication" + fi - # Set fixed ports for production (instead of random ports used in dev) - sed -i "s/^# CADDY_ADMIN_PORT=.*/CADDY_ADMIN_PORT=2019/" "${TMP_DIR}/config" - sed -i "s/^# INTERNAL_DNS_PORT=.*/INTERNAL_DNS_PORT=5353/" "${TMP_DIR}/config" + # Set fixed ports for production (nested under caddy:) + # Uncomment the caddy block and set ports + if grep -q '^# caddy:' "${TMP_DIR}/config.yaml"; then + sed -i "s|^# caddy:|caddy:|" "${TMP_DIR}/config.yaml" + sed -i "s|^# admin_port:.*| admin_port: 2019|" "${TMP_DIR}/config.yaml" + sed -i "s|^# internal_dns_port:.*| internal_dns_port: 5353|" "${TMP_DIR}/config.yaml" + fi info "Installing config file at ${CONFIG_FILE}..." - $SUDO install -m 640 "${TMP_DIR}/config" "$CONFIG_FILE" + $SUDO install -m 640 "${TMP_DIR}/config.yaml" "$CONFIG_FILE" $SUDO chown root:root "$CONFIG_FILE" fi else info "Config file already exists at ${CONFIG_FILE}, skipping..." + # Read JWT_SECRET from existing config for CLI token generation + # Handle: leading whitespace, single/double quotes, trailing whitespace + JWT_SECRET=$($SUDO grep -E '^[[:space:]]*jwt_secret[[:space:]]*:' "$CONFIG_FILE" 2>/dev/null | head -1 | sed 's/^[[:space:]]*jwt_secret[[:space:]]*:[[:space:]]*//' | tr -d "\"'" | sed 's/[[:space:]]*$//' || true) fi # ============================================================================= @@ -514,21 +528,6 @@ if [ "$OS" = "darwin" ]; then info "Installing launchd service..." - # Build environment variables from config file - ENV_DICT="" - if [ -f "$CONFIG_FILE" ]; then - while IFS= read -r line; do - # Skip comments and empty lines - [[ "$line" =~ ^[[:space:]]*# ]] && continue - [[ -z "$line" ]] && continue - key="${line%%=*}" - value="${line#*=}" - ENV_DICT="${ENV_DICT} - ${key} - ${value}" - done < "$CONFIG_FILE" - fi - cat > "$PLIST_PATH" << PLIST @@ -543,7 +542,9 @@ if [ "$OS" = "darwin" ]; then EnvironmentVariables PATH - /opt/homebrew/opt/e2fsprogs/sbin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${ENV_DICT} + /opt/homebrew/opt/e2fsprogs/sbin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + CONFIG_PATH + ${CONFIG_FILE} KeepAlive @@ -571,7 +572,7 @@ After=network.target [Service] Type=simple Environment="HOME=${DATA_DIR}" -EnvironmentFile=${CONFIG_FILE} +Environment="CONFIG_PATH=${CONFIG_FILE}" ExecStart=${INSTALL_DIR}/${BINARY_NAME} Restart=on-failure RestartSec=5 @@ -629,64 +630,120 @@ fi # ============================================================================= CLI_REPO="kernel/hypeman-cli" +CLI_INSTALLED=false -# CLI releases use goreleaser naming: "macos" not "darwin", .zip not .tar.gz on macOS -if [ "$OS" = "darwin" ]; then - CLI_OS="macos" - CLI_EXT="zip" +if [ -n "$CLI_BRANCH" ]; then + # Build CLI from source + info "Building CLI from source (branch: $CLI_BRANCH)..." + command -v go >/dev/null 2>&1 || error "go is required for CLI_BRANCH mode but not installed" + + CLI_BUILD_DIR="${TMP_DIR}/hypeman-cli" + if ! git clone --branch "$CLI_BRANCH" --depth 1 -q "https://github.com/${CLI_REPO}.git" "$CLI_BUILD_DIR" 2>&1; then + error "Failed to clone CLI repository (branch: $CLI_BRANCH)" + fi + + info "Compiling CLI..." + mkdir -p "${TMP_DIR}/cli-bin" + if ! (cd "$CLI_BUILD_DIR" && go build -o "${TMP_DIR}/cli-bin/hypeman" ./cmd/hypeman 2>&1); then + error "Failed to build CLI from source" + fi + + info "Installing hypeman CLI to ${INSTALL_DIR}..." + $SUDO install -m 755 "${TMP_DIR}/cli-bin/hypeman" "${INSTALL_DIR}/hypeman" + if [ "$OS" != "darwin" ]; then + info "Linking hypeman to /usr/local/bin..." + $SUDO ln -sf "${INSTALL_DIR}/hypeman" /usr/local/bin/hypeman + fi + CLI_INSTALLED=true else - CLI_OS="$OS" - CLI_EXT="tar.gz" -fi + # Download CLI from release + # CLI releases use goreleaser naming: "macos" not "darwin", .zip not .tar.gz on macOS + if [ "$OS" = "darwin" ]; then + CLI_OS="macos" + CLI_EXT="zip" + else + CLI_OS="$OS" + CLI_EXT="tar.gz" + fi -if [ -z "$CLI_VERSION" ] || [ "$CLI_VERSION" == "latest" ]; then - info "Fetching latest CLI version with available artifacts..." - CLI_VERSION=$(find_release_with_artifact "$CLI_REPO" "hypeman" "$CLI_OS" "$ARCH" "$CLI_EXT" || true) - if [ -z "$CLI_VERSION" ]; then - warn "Failed to find a CLI release with artifacts for ${CLI_OS}/${ARCH}, skipping CLI installation" + if [ -z "$CLI_VERSION" ] || [ "$CLI_VERSION" == "latest" ]; then + info "Fetching latest CLI version with available artifacts..." + CLI_VERSION=$(find_release_with_artifact "$CLI_REPO" "hypeman" "$CLI_OS" "$ARCH" "$CLI_EXT" || true) + if [ -z "$CLI_VERSION" ]; then + warn "Failed to find a CLI release with artifacts for ${CLI_OS}/${ARCH}, skipping CLI installation" + fi fi -fi -if [ -n "$CLI_VERSION" ]; then - info "Installing Hypeman CLI version: $CLI_VERSION" + if [ -n "$CLI_VERSION" ]; then + info "Installing Hypeman CLI version: $CLI_VERSION" + + CLI_VERSION_NUM="${CLI_VERSION#v}" + CLI_ARCHIVE_NAME="hypeman_${CLI_VERSION_NUM}_${CLI_OS}_${ARCH}.${CLI_EXT}" + CLI_DOWNLOAD_URL="https://github.com/${CLI_REPO}/releases/download/${CLI_VERSION}/${CLI_ARCHIVE_NAME}" + + info "Downloading CLI ${CLI_ARCHIVE_NAME}..." + if curl -fsSL "$CLI_DOWNLOAD_URL" -o "${TMP_DIR}/${CLI_ARCHIVE_NAME}"; then + info "Extracting CLI..." + mkdir -p "${TMP_DIR}/cli" + if [ "$CLI_EXT" = "zip" ]; then + unzip -qo "${TMP_DIR}/${CLI_ARCHIVE_NAME}" -d "${TMP_DIR}/cli" + else + tar -xzf "${TMP_DIR}/${CLI_ARCHIVE_NAME}" -C "${TMP_DIR}/cli" + fi - CLI_VERSION_NUM="${CLI_VERSION#v}" - CLI_ARCHIVE_NAME="hypeman_${CLI_VERSION_NUM}_${CLI_OS}_${ARCH}.${CLI_EXT}" - CLI_DOWNLOAD_URL="https://github.com/${CLI_REPO}/releases/download/${CLI_VERSION}/${CLI_ARCHIVE_NAME}" + if [ "$OS" = "darwin" ]; then + info "Installing hypeman CLI to ${INSTALL_DIR}..." + $SUDO install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman" + else + info "Installing hypeman CLI to ${INSTALL_DIR}..." + $SUDO install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman" - info "Downloading CLI ${CLI_ARCHIVE_NAME}..." - if curl -fsSL "$CLI_DOWNLOAD_URL" -o "${TMP_DIR}/${CLI_ARCHIVE_NAME}"; then - info "Extracting CLI..." - mkdir -p "${TMP_DIR}/cli" - if [ "$CLI_EXT" = "zip" ]; then - unzip -qo "${TMP_DIR}/${CLI_ARCHIVE_NAME}" -d "${TMP_DIR}/cli" + info "Linking hypeman to /usr/local/bin..." + $SUDO ln -sf "${INSTALL_DIR}/hypeman" /usr/local/bin/hypeman + fi + CLI_INSTALLED=true else - tar -xzf "${TMP_DIR}/${CLI_ARCHIVE_NAME}" -C "${TMP_DIR}/cli" + warn "Failed to download CLI from ${CLI_DOWNLOAD_URL}, skipping CLI installation" fi + fi +fi - if [ "$OS" = "darwin" ]; then - info "Installing hypeman CLI to ${INSTALL_DIR}..." - $SUDO install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman" - else - # Install CLI binary - info "Installing hypeman CLI to ${INSTALL_DIR}..." - $SUDO install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman-cli" +# Generate CLI config file with a pre-authenticated token +if [ "$CLI_INSTALLED" = true ]; then + CLI_CONFIG_DIR="$HOME/.config/hypeman" + CLI_CONFIG_FILE="${CLI_CONFIG_DIR}/cli.yaml" + if [ ! -f "$CLI_CONFIG_FILE" ]; then + info "Generating CLI configuration..." + mkdir -p "$CLI_CONFIG_DIR" + + # Determine the port from config + CLI_PORT="8080" + if [ -f "$CONFIG_FILE" ]; then + PARSED_PORT=$(grep -E '^[[:space:]]*port[[:space:]]*:' "$CONFIG_FILE" 2>/dev/null | head -1 | sed 's/^[[:space:]]*port[[:space:]]*:[[:space:]]*//' | tr -d "\"'" | sed 's/[[:space:]]*$//' || true) + if [ -n "$PARSED_PORT" ]; then + CLI_PORT="$PARSED_PORT" + fi + fi - # Install wrapper script to /usr/local/bin for PATH access - info "Installing hypeman wrapper to /usr/local/bin..." - $SUDO tee /usr/local/bin/hypeman > /dev/null << WRAPPER -#!/bin/bash -# Wrapper script for hypeman CLI that auto-generates API token -set -a -source ${CONFIG_FILE} -set +a -export HYPEMAN_API_KEY=\$(${INSTALL_DIR}/hypeman-token -user-id "cli-user-\$(whoami)" 2>/dev/null) -exec ${INSTALL_DIR}/hypeman-cli "\$@" -WRAPPER - $SUDO chmod 755 /usr/local/bin/hypeman + # Generate a long-lived CLI token + # Pass JWT_SECRET explicitly since the config file may not be readable by the current user + CLI_TOKEN=$(JWT_SECRET="${JWT_SECRET}" "${INSTALL_DIR}/hypeman-token" -user-id "cli-$(whoami)" -duration 8760h 2>/dev/null || true) + if [ -z "$CLI_TOKEN" ]; then + warn "Failed to generate CLI token. You may need to run: hypeman-token -user-id cli-$(whoami) > token and add it to ${CLI_CONFIG_FILE}" + cat > "$CLI_CONFIG_FILE" << CLIEOF +base_url: http://localhost:${CLI_PORT} +api_key: "" +CLIEOF + else + cat > "$CLI_CONFIG_FILE" << CLIEOF +base_url: http://localhost:${CLI_PORT} +api_key: "${CLI_TOKEN}" +CLIEOF fi + chmod 600 "$CLI_CONFIG_FILE" + info "CLI configured at ${CLI_CONFIG_FILE}" else - warn "Failed to download CLI from ${CLI_DOWNLOAD_URL}, skipping CLI installation" + info "CLI config already exists at ${CLI_CONFIG_FILE}, skipping..." fi fi @@ -712,7 +769,8 @@ if [ "$OS" = "darwin" ]; then echo " API Binary: ${INSTALL_DIR}/${BINARY_NAME}" echo " CLI: ${INSTALL_DIR}/hypeman" echo " Token tool: ${INSTALL_DIR}/hypeman-token" - echo " Config: ${CONFIG_FILE}" + echo " Server config: ${CONFIG_FILE}" + echo " CLI config: ~/.config/hypeman/cli.yaml" echo " Data: ${DATA_DIR}" echo " Service: ~/Library/LaunchAgents/com.kernel.hypeman.plist" echo " Logs: ${DATA_DIR}/logs/hypeman.log" @@ -720,7 +778,8 @@ else echo " API Binary: ${INSTALL_DIR}/${BINARY_NAME}" echo " CLI: /usr/local/bin/hypeman" echo " Token tool: /usr/local/bin/hypeman-token" - echo " Config: ${CONFIG_FILE}" + echo " Server config: ${CONFIG_FILE}" + echo " CLI config: ~/.config/hypeman/cli.yaml" echo " Data: ${DATA_DIR}" echo " Service: ${SERVICE_NAME}.service" fi @@ -728,7 +787,7 @@ fi echo "" echo "" echo "Next steps:" -echo " - (Optional) Edit ${CONFIG_FILE} to configure your installation" +echo " - (Optional) Edit ${CONFIG_FILE} to configure your server" echo "" echo "Get Started:" echo "╭──────────────────────────────────────────╮" diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index ccf9d673..39dd0eeb 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -123,7 +123,7 @@ else fi # ============================================================================= -# Remove binaries and wrappers +# Remove binaries and symlinks # ============================================================================= info "Removing binaries..." @@ -133,7 +133,7 @@ if [ "$OS" = "darwin" ]; then $SUDO rm -f "${INSTALL_DIR}/hypeman-token" $SUDO rm -f "${INSTALL_DIR}/hypeman" else - # Remove wrapper scripts from /usr/local/bin + # Remove symlinks from /usr/local/bin $SUDO rm -f /usr/local/bin/hypeman $SUDO rm -f /usr/local/bin/hypeman-token