diff options
| author | Paul Buetow <paul@buetow.org> | 2026-03-25 10:43:43 +0200 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2026-03-25 10:43:43 +0200 |
| commit | ef53a98c39c26d69b4bfd3a4e925050b220a02c9 (patch) | |
| tree | d6e747f4a9eea844f498b3f807567d3a5330694e /lib/hyperstack | |
| parent | 917c3d9a777d343b422599f291f242f4bf025ba0 (diff) | |
hyperstack: split 3335-line monolith into lib/hyperstack/ modules
Extracts all classes from hyperstack.rb into focused library files:
- lib/hyperstack/config.rb — ConfigLoader + Config (TOML loading, validation)
- lib/hyperstack/state.rb — StateStore + PrefixedOutput (JSON state, threaded output)
- lib/hyperstack/client.rb — HyperstackClient (REST API + retry logic)
- lib/hyperstack/wireguard.rb — LocalWireGuard (wg1.conf peer management, /etc/hosts)
- lib/hyperstack/provisioning.rb — ProvisioningScripts + RemoteProvisioner (SSH bootstrap)
- lib/hyperstack/manager.rb — Manager (VM lifecycle orchestration)
- lib/hyperstack/watcher.rb — VllmWatcher (Prometheus + GPU dashboard)
- lib/hyperstack/cli.rb — CLI (OptionParser command dispatch)
hyperstack.rb becomes a 46-line entry point with require_relative calls.
All files pass `ruby -c` syntax check and `--help` runs correctly.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'lib/hyperstack')
| -rw-r--r-- | lib/hyperstack/cli.rb | 330 | ||||
| -rw-r--r-- | lib/hyperstack/client.rb | 140 | ||||
| -rw-r--r-- | lib/hyperstack/config.rb | 665 | ||||
| -rw-r--r-- | lib/hyperstack/manager.rb | 1057 | ||||
| -rw-r--r-- | lib/hyperstack/provisioning.rb | 458 | ||||
| -rw-r--r-- | lib/hyperstack/state.rb | 57 | ||||
| -rw-r--r-- | lib/hyperstack/watcher.rb | 425 | ||||
| -rw-r--r-- | lib/hyperstack/wireguard.rb | 234 |
8 files changed, 3366 insertions, 0 deletions
diff --git a/lib/hyperstack/cli.rb b/lib/hyperstack/cli.rb new file mode 100644 index 0000000..f575b59 --- /dev/null +++ b/lib/hyperstack/cli.rb @@ -0,0 +1,330 @@ +# frozen_string_literal: true + +require 'optparse' + +module HyperstackVM + class CLI + def initialize(argv) + @argv = argv.dup + @config_path = File.join(__dir__, 'hyperstack-vm.toml') + @config_explicit = false + end + + def show_help + puts @global_parser + puts + puts 'Commands:' + puts ' create [--replace] [--dry-run] [--vllm|--no-vllm] [--ollama|--no-ollama] [--model PRESET]' + puts ' create-both [--replace] [--dry-run] [--vllm|--no-vllm] [--ollama|--no-ollama]' + puts ' Provision hyperstack-vm1-gptoss.toml and hyperstack-vm2.toml concurrently.' + puts ' WireGuard setup is serialized: VM1 writes the base wg1.conf first,' + puts ' then VM2 adds its peer. Requires both TOML files next to the script.' + puts ' delete [--vm-id ID] [--dry-run]' + puts ' delete-both [--dry-run]' + puts ' Delete the VMs tracked by hyperstack-vm1-gptoss.toml and hyperstack-vm2.toml.' + puts ' status' + puts ' watch' + puts ' Poll all active VMs for vLLM and GPU stats every 60 s.' + puts ' test' + puts ' model list' + puts ' model switch PRESET [--dry-run]' + end + + def run + @global_parser = OptionParser.new do |opts| + opts.banner = 'Usage: ruby hyperstack.rb [--config path] <create|delete|status> [options]' + opts.on('--config PATH', "Path to TOML config (default: #{@config_path})") do |value| + @config_path = value + @config_explicit = true + end + opts.on('-h', '--help', 'Show help') do + show_help + exit 0 + end + end + @global_parser.order!(@argv) + + command = @argv.shift + if command.nil? + show_help + exit 0 + end + + # create-both loads its own config files and does not use the default config path. + # Parse it before building the manager so we avoid loading the default config needlessly. + if command == 'create-both' + opts = parse_create_options(@argv, include_model_preset: false) + run_create_both(**opts) + return + end + + if command == 'delete-both' + opts = parse_delete_both_options(@argv) + run_delete_both(**opts) + return + end + + if command == 'status' + run_status + return + end + + if command == 'watch' + run_watch + return + end + + # All other commands operate on a single VM defined by the --config path. + config_loader = ConfigLoader.load(@config_path) + manager = build_manager(config_loader.config) + + case command + when 'create' + opts = parse_create_options(@argv) + manager.create(**opts) + when 'delete' + vm_id = nil + dry_run = false + parser = OptionParser.new do |opts| + opts.on('--vm-id ID', Integer, 'Delete a VM by ID instead of using the local state file') do |value| + vm_id = value + end + opts.on('--dry-run', 'Show which VM would be deleted without deleting it') { dry_run = true } + end + parser.parse!(@argv) + manager.delete(vm_id: vm_id, dry_run: dry_run) + when 'test' + manager.test + when 'model' + sub = @argv.shift + raise Error, 'Missing model subcommand. Use: model list | model switch PRESET [--dry-run]' if sub.nil? + + case sub + when 'list' + manager.list_models + when 'switch' + preset = @argv.shift + raise Error, 'Missing preset name. Usage: model switch PRESET [--dry-run]' if preset.nil? + + dry_run = false + OptionParser.new { |o| o.on('--dry-run') { dry_run = true } }.parse!(@argv) + manager.switch_model(preset_name: preset, dry_run: dry_run) + else + raise Error, "Unknown model subcommand #{sub.inspect}. Use list or switch." + end + else + raise Error, + "Unknown command #{command.inspect}. Use create, create-both, delete, delete-both, status, watch, test, or model." + end + end + + private + + # Parses the shared --replace / --dry-run / --vllm / --ollama / --model flags + # used by both 'create' and 'create-both'. When include_model_preset is false + # (create-both), the --model flag is not registered because each VM uses its own + # TOML default. Returns a hash suitable for splatting into Manager#create. + def parse_create_options(argv, include_model_preset: true) + opts = { replace: false, dry_run: false, install_vllm: nil, install_ollama: nil, install_comfyui: nil, + vllm_preset: nil } + OptionParser.new do |o| + o.on('--replace', 'Delete the tracked VM before creating a new one') { opts[:replace] = true } + o.on('--dry-run', 'Print the create plan without creating a VM') { opts[:dry_run] = true } + o.on('--vllm', 'Enable vLLM setup (overrides config)') { opts[:install_vllm] = true } + o.on('--no-vllm', 'Disable vLLM setup (overrides config)') { opts[:install_vllm] = false } + o.on('--ollama', 'Enable Ollama setup (overrides config)') { opts[:install_ollama] = true } + o.on('--no-ollama', 'Disable Ollama setup (overrides config)') { opts[:install_ollama] = false } + o.on('--comfyui', 'Enable ComfyUI setup (overrides config)') { opts[:install_comfyui] = true } + o.on('--no-comfyui', 'Disable ComfyUI setup (overrides config)') { opts[:install_comfyui] = false } + if include_model_preset + o.on('--model PRESET', 'Use a named vLLM preset at create time') do |v| + opts[:vllm_preset] = v + end + end + end.parse!(argv) + opts + end + + def parse_delete_both_options(argv) + opts = { dry_run: false } + OptionParser.new do |o| + o.on('--dry-run', 'Show which VMs would be deleted without deleting them') { opts[:dry_run] = true } + end.parse!(argv) + opts + end + + # Constructs a Manager and all its dependencies from a Config object. + # Accepts optional output destination and WireGuard concurrency hooks. + def build_manager(config, out: $stdout, wg_setup_pre: nil, wg_setup_post: nil) + state_store = StateStore.new(config.state_file) + client = HyperstackClient.new(base_url: config.api_base_url, api_key: config.api_key) + local_wireguard = build_local_wireguard(config) + Manager.new( + config: config, + client: client, + state_store: state_store, + local_wireguard: local_wireguard, + out: out, + wg_setup_pre: wg_setup_pre, + wg_setup_post: wg_setup_post + ) + end + + def build_local_wireguard(config) + LocalWireGuard.new( + interface_name: config.local_interface_name, + config_path: config.local_wg_config_path + ) + end + + # Starts the VllmWatcher dashboard for all active VMs. + # Reuses status_config_loaders so it auto-discovers the same set of VMs + # that `status` would show (honours --config if given explicitly). + def run_watch + loaders = status_config_loaders + raise Error, 'No active VMs found. Run `create` or `create-both` first.' if loaders.empty? + + VllmWatcher.new(config_loaders: loaders).run + end + + def run_status + loaders = status_config_loaders + if loaders.one? + build_manager(loaders.first.config).status + return + end + + expected_ips = [] + loaders.each_with_index do |loader, index| + puts if index.positive? + puts "[#{File.basename(loader.path)}]" + expected_ip = build_manager(loader.config).status(include_local_wireguard: false) + expected_ips << expected_ip if expected_ip + end + + puts + puts '[local-wireguard]' + build_manager(loaders.first.config).show_local_wireguard(expected_ips) + end + + def status_config_loaders + return [ConfigLoader.load(@config_path)] if @config_explicit + + candidates = [ + @config_path, + File.join(__dir__, 'hyperstack-vm1-gptoss.toml'), + File.join(__dir__, 'hyperstack-vm2.toml'), + File.join(__dir__, 'hyperstack-vm-photo.toml') + ].uniq.select { |path| File.exist?(path) } + + loaders = candidates.map { |path| ConfigLoader.load(path) } + tracked = loaders.select { |loader| File.exist?(loader.config.state_file) } + tracked.empty? ? [ConfigLoader.load(@config_path)] : tracked + end + + def pair_config_loaders + [ + ConfigLoader.load(File.join(__dir__, 'hyperstack-vm1-gptoss.toml')), + ConfigLoader.load(File.join(__dir__, 'hyperstack-vm2.toml')) + ] + end + + # Provisions hyperstack-vm1 and hyperstack-vm2 concurrently in separate threads. + # WireGuard setup is serialized: VM1 runs first (replacing the base wg1.conf), then + # VM2 adds its peer. A Mutex+ConditionVariable acts as a one-shot latch between threads. + # If VM1 fails before reaching the WG step the latch is still released so VM2 doesn't hang. + # vllm_preset is accepted but ignored — each VM uses its own TOML default preset. + def run_create_both(replace:, dry_run:, install_vllm:, install_ollama:, install_comfyui: nil, vllm_preset: nil) # rubocop:disable Lint/UnusedMethodArgument + vm1_loader, vm2_loader = pair_config_loaders + vm1_config = vm1_loader.config + vm2_config = vm2_loader.config + + out_mutex = Mutex.new + wg_mutex = Mutex.new + wg_cv = ConditionVariable.new + vm1_wg_state = { done: false, error: nil } + + # VM1 signals the latch after its WG step (whether WG ran or was already done). + vm1_wg_post = proc do + wg_mutex.synchronize do + vm1_wg_state[:done] = true + wg_cv.broadcast + end + end + + # VM2 blocks here until VM1's WG step resolves, then raises if VM1 failed. + vm2_wg_pre = proc do + wg_mutex.synchronize { wg_cv.wait(wg_mutex) until vm1_wg_state[:done] || vm1_wg_state[:error] } + raise Error, 'VM1 WireGuard setup failed; cannot add VM2 peer.' if vm1_wg_state[:error] + end + + manager1 = build_manager(vm1_config, + out: PrefixedOutput.new('[vm1] ', $stdout, out_mutex), + wg_setup_post: vm1_wg_post) + manager2 = build_manager(vm2_config, + out: PrefixedOutput.new('[vm2] ', $stdout, out_mutex), + wg_setup_pre: vm2_wg_pre) + + errors = {} + create_opts = { replace: replace, dry_run: dry_run, + install_vllm: install_vllm, install_ollama: install_ollama, install_comfyui: install_comfyui } + + vm1_thread = Thread.new do + manager1.create(**create_opts) + rescue Error => e + errors[:vm1] = e.message + # Unblock VM2 even if VM1 failed so the process doesn't hang. + wg_mutex.synchronize do + vm1_wg_state[:error] = e.message + wg_cv.broadcast + end + end + + vm2_thread = Thread.new do + manager2.create(**create_opts) + rescue Error => e + errors[:vm2] = e.message + end + + [vm1_thread, vm2_thread].each(&:join) + + errors.each { |vm, msg| warn("ERROR [#{vm}]: #{msg}") } + exit 1 unless errors.empty? + end + + def run_delete_both(dry_run:) + out_mutex = Mutex.new + errors_mutex = Mutex.new + errors = {} + loaders = pair_config_loaders + local_wg_out = PrefixedOutput.new('[local-wireguard] ', $stdout, out_mutex) + threads = loaders.each_with_index.map do |loader, index| + label = "vm#{index + 1}" + manager = build_manager(loader.config, out: PrefixedOutput.new("[#{label}] ", $stdout, out_mutex)) + + Thread.new do + manager.delete(dry_run: dry_run, skip_local_cleanup: true) + rescue Error => e + errors_mutex.synchronize { errors[label.to_sym] = e.message } + end + end + threads.each(&:join) + + if errors.empty? + allowed_ips = loaders.map { |loader| "#{loader.config.wireguard_gateway_ip}/32" } + hostnames = loaders.map { |loader| loader.config.wireguard_gateway_hostname } + begin + local_manager = build_manager(loaders.first.config, out: local_wg_out) + cleanup = local_manager.send(:cleanup_local_access, dry_run: dry_run, hostnames: hostnames, + allowed_ips: allowed_ips) + local_manager.send(:report_local_cleanup, local_wg_out, cleanup, dry_run: dry_run) + rescue Error => e + errors[:local_wireguard] = e.message + end + end + + errors.each { |vm, msg| warn("ERROR [#{vm}]: #{msg}") } + exit 1 unless errors.empty? + end + end +end diff --git a/lib/hyperstack/client.rb b/lib/hyperstack/client.rb new file mode 100644 index 0000000..b7b4a6b --- /dev/null +++ b/lib/hyperstack/client.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'openssl' +require 'socket' +require 'timeout' + +module HyperstackVM + # HTTP client for the Hyperstack (NexGenCloud) REST API. + # Handles authentication, JSON encoding/decoding, and retry logic with exponential back-off. + class HyperstackClient + def initialize(base_url:, api_key:) + @base_uri = URI(base_url) + @api_key = api_key + end + + def list_environments + response = request(:get, '/core/environments') + response.fetch('environments', []) + end + + def list_keypairs + response = request(:get, '/core/keypairs') + response.fetch('keypairs', []) + end + + def list_flavors + response = request(:get, '/core/flavors') + Array(response['data']).flat_map do |entry| + Array(entry['flavors']).map do |flavor| + flavor.merge( + 'region_name' => flavor['region_name'] || entry['region_name'], + 'gpu' => flavor['gpu'] || entry['gpu'] + ) + end + end + end + + def list_images + response = request(:get, '/core/images') + Array(response['images']).flat_map do |entry| + Array(entry['images']).map do |image| + image.merge( + 'region_name' => image['region_name'] || entry['region_name'], + 'type' => image['type'] || entry['type'] + ) + end + end + end + + def list_vms + response = request(:get, '/core/virtual-machines') + response.fetch('instances', []) + end + + def get_vm(vm_id) + response = request(:get, "/core/virtual-machines/#{vm_id}") + response.fetch('instance', nil) + end + + def create_vm(payload) + request(:post, '/core/virtual-machines', payload) + end + + def delete_vm(vm_id) + request(:delete, "/core/virtual-machines/#{vm_id}") + end + + def create_vm_rule(vm_id, payload) + request(:post, "/core/virtual-machines/#{vm_id}/sg-rules", payload) + end + + def delete_vm_rule(vm_id, rule_id) + request(:delete, "/core/virtual-machines/#{vm_id}/sg-rules/#{rule_id}") + end + + private + + def request(method, path, payload = nil) + uri = @base_uri.dup + uri.path = "#{@base_uri.path}#{path}" + + request = case method + when :get + Net::HTTP::Get.new(uri) + when :post + Net::HTTP::Post.new(uri) + when :delete + Net::HTTP::Delete.new(uri) + else + raise Error, "Unsupported HTTP method: #{method}" + end + + request['accept'] = 'application/json' + request['api_key'] = @api_key + if payload + request['content-type'] = 'application/json' + request.body = JSON.generate(payload) + end + + retries_left = 4 + begin + response = Net::HTTP.start( + uri.host, + uri.port, + use_ssl: uri.scheme == 'https', + open_timeout: 30, + read_timeout: 120 + ) { |http| http.request(request) } + + parse_response(response) + rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, + Errno::EHOSTUNREACH, Errno::ENETUNREACH, + SocketError, OpenSSL::SSL::SSLError, Net::OpenTimeout => e + raise Error, "Hyperstack API request failed for #{path}: #{e.message}" if retries_left <= 0 + + retries_left -= 1 + delay = (4 - retries_left) * 5 + warn "API request to #{path} failed (#{e.class}: #{e.message}), retrying in #{delay}s (#{retries_left} left)..." + sleep delay + retry + end + end + + def parse_response(response) + body = response.body.to_s + payload = body.empty? ? {} : JSON.parse(body) + + if response.code.to_i >= 400 || payload['status'] == false + message = payload['message'] || payload['error_reason'] || response.message + raise Error, "Hyperstack API error (HTTP #{response.code}): #{message}" + end + + payload + rescue JSON::ParserError => e + raise Error, "Failed to parse Hyperstack API response: #{e.message}" + end + end +end diff --git a/lib/hyperstack/config.rb b/lib/hyperstack/config.rb new file mode 100644 index 0000000..402f45d --- /dev/null +++ b/lib/hyperstack/config.rb @@ -0,0 +1,665 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'ipaddr' +require 'json' +require 'toml-rb' + +module HyperstackVM + class ConfigLoader + attr_reader :path + + def self.load(path) + expanded = File.expand_path(path) + raise Error, "Config file not found: #{expanded}" unless File.exist?(expanded) + + raw = TomlRB.load_file(expanded) + new(raw, expanded) + rescue TomlRB::ParseError => e + raise Error, "Failed to parse TOML config #{expanded}: #{e.message}" + end + + def initialize(raw, path) + @path = path + @data = deep_merge(DEFAULTS, raw || {}) + validate! + end + + def config + Config.new(@data, @path) + end + + private + + DEFAULTS = { + 'auth' => { + 'api_key_file' => '~/.hyperstack' + }, + 'hyperstack' => { + 'base_url' => 'https://infrahub-api.nexgencloud.com/v1' + }, + 'state' => { + 'file' => '.hyperstack-vm-state.json' + }, + 'vm' => { + 'name_prefix' => 'hyperstack', + 'hostname' => 'hyperstack', + 'flavor_name' => 'n3-A100x1', + 'image_name' => 'Ubuntu Server 24.04 LTS R570 CUDA 12.8 with Docker', + 'assign_floating_ip' => true, + 'create_bootable_volume' => false, + 'enable_port_randomization' => false, + 'labels' => %w[gpt-oss-120b wireguard] + }, + 'ssh' => { + 'username' => 'ubuntu', + 'private_key_path' => '~/.ssh/id_rsa', + 'hyperstack_key_name' => 'earth', + 'port' => 22, + 'connect_timeout_sec' => 10 + }, + 'network' => { + 'wireguard_udp_port' => 56_710, + 'wireguard_subnet' => '192.168.3.0/24', + # Optional: explicit server-side WireGuard IP. When nil, derived as subnet + 1 (i.e. .1). + # Set to a different address (e.g. 192.168.3.3) for a second VM sharing the same wg1 tunnel. + 'wireguard_server_ip' => nil, + 'ollama_port' => 11_434, + 'allowed_ssh_cidrs' => ['auto'], + 'allowed_wireguard_cidrs' => ['auto'] + }, + 'bootstrap' => { + 'enable_guest_bootstrap' => true, + 'install_wireguard' => true, + 'configure_ufw' => true, + 'configure_ollama_host' => false + }, + 'ollama' => { + 'install' => false, + 'models_dir' => '/ephemeral/ollama/models', + 'listen_host' => '0.0.0.0:11434', + 'gpu_overhead_mb' => 2000, + 'num_parallel' => 1, + 'context_length' => 32_768, + 'pull_models' => ['qwen3-coder:30b', 'gpt-oss:20b', 'gpt-oss:120b', 'nemotron-3-super'] + }, + 'vllm' => { + 'install' => true, + 'model' => 'bullpoint/Qwen3-Coder-Next-AWQ-4bit', + 'hug_cache_dir' => '/ephemeral/hug', + 'container_name' => 'vllm_qwen3', + 'max_model_len' => 262_144, + 'gpu_memory_utilization' => 0.92, + 'tensor_parallel_size' => 1, + 'tool_call_parser' => 'qwen3_coder' + }, + 'comfyui' => { + 'install' => false, + 'port' => 8188, + 'models_dir' => '/ephemeral/comfyui/models', + 'output_dir' => '/ephemeral/comfyui/output', + 'container_name' => 'comfyui', + # Models to pre-download: Real-ESRGAN for fast upscaling, SUPIR for deep restoration. + 'models' => [] + }, + 'wireguard' => { + 'auto_setup' => true, + 'setup_script' => './wg1-setup.sh' + }, + 'local_client' => { + 'check_wg1_service' => true, + 'interface_name' => 'wg1', + 'config_path' => '/etc/wireguard/wg1.conf' + } + }.freeze + + def validate! + %w[auth hyperstack state vm ssh network bootstrap ollama vllm comfyui wireguard local_client].each do |section| + raise Error, "Missing config section [#{section}]" unless @data.key?(section) + end + + %w[environment_name flavor_name image_name].each do |key| + raise Error, "Missing [vm].#{key} in config #{path}" if blank?(dig('vm', key)) + end + + if fetch('vm', 'hostname') && fetch('vm', 'hostname') !~ /\A[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\z/ + raise Error, + "Invalid [vm].hostname #{fetch('vm', + 'hostname').inspect}; use lowercase letters, digits, and hyphens only." + end + + %w[username hyperstack_key_name].each do |key| + raise Error, "Missing [ssh].#{key} in config #{path}" if blank?(dig('ssh', key)) + end + + ssh_cidrs = normalized_cidrs(fetch('network', 'allowed_ssh_cidrs')) + wireguard_cidrs = normalized_cidrs(fetch('network', 'allowed_wireguard_cidrs')) + + raise Error, missing_cidr_message('allowed_ssh_cidrs') if ssh_cidrs.empty? + raise Error, missing_cidr_message('allowed_wireguard_cidrs') if wireguard_cidrs.empty? + + [fetch('network', 'wireguard_subnet'), *ssh_cidrs, *wireguard_cidrs].each do |cidr| + next if cidr == 'auto' + + IPAddr.new(cidr) + rescue IPAddr::InvalidAddressError => e + raise Error, "Invalid CIDR #{cidr.inspect}: #{e.message}" + end + + server_ip = fetch('network', 'wireguard_server_ip') + return unless server_ip + + # Validate that the explicit server WireGuard IP is within the configured subnet. + begin + subnet = IPAddr.new(fetch('network', 'wireguard_subnet')) + unless subnet.include?(IPAddr.new(server_ip)) + raise Error, + "wireguard_server_ip #{server_ip.inspect} is not in wireguard_subnet #{fetch('network', + 'wireguard_subnet')}" + end + rescue IPAddr::InvalidAddressError => e + raise Error, "Invalid wireguard_server_ip #{server_ip.inspect}: #{e.message}" + end + end + + def fetch(section, key) + dig(section, key) + end + + def dig(*keys) + keys.reduce(@data) do |memo, key| + memo.is_a?(Hash) ? memo[key] : nil + end + end + + def blank?(value) + value.nil? || value.to_s.strip.empty? + end + + def truthy?(value) + value == true + end + + def normalized_cidrs(values) + Array(values).map { |value| value.to_s.strip }.reject(&:empty?) + end + + def missing_cidr_message(key) + "Missing [network].#{key} in config #{path}; set it to one or more CIDRs, or ['auto'] to restrict access to the current public operator IP." + end + + def deep_merge(left, right) + left.merge(right) do |_key, old_value, new_value| + if old_value.is_a?(Hash) && new_value.is_a?(Hash) + deep_merge(old_value, new_value) + else + new_value + end + end + end + end + + class Config + attr_reader :path + + def initialize(data, path = nil) + @data = data + @path = path + end + + def api_key + key_path = expand_path(fetch('auth', 'api_key_file')) + raise Error, "API key file not found: #{key_path}" unless File.exist?(key_path) + + token = File.readlines(key_path, chomp: true).find { |line| !line.strip.empty? }&.strip + raise Error, "API key file is empty: #{key_path}" if token.nil? || token.empty? + + token + rescue Errno::EACCES => e + raise Error, "Cannot read API key file #{key_path}: #{e.message}" + end + + def api_base_url + fetch('hyperstack', 'base_url') + end + + def state_file + expand_path(fetch('state', 'file')) + end + + def environment_name + fetch('vm', 'environment_name') + end + + def flavor_name + fetch('vm', 'flavor_name') + end + + def image_name + fetch('vm', 'image_name') + end + + def vm_name_prefix + fetch('vm', 'name_prefix') + end + + def generated_vm_name + "#{vm_name_prefix}-#{Time.now.utc.strftime('%Y%m%d%H%M%S')}" + end + + def vm_hostname + value = fetch('vm', 'hostname') + return nil if blank?(value) + + value.to_s.downcase + end + + def assign_floating_ip? + truthy?(fetch('vm', 'assign_floating_ip')) + end + + def create_bootable_volume? + truthy?(fetch('vm', 'create_bootable_volume')) + end + + def enable_port_randomization? + truthy?(fetch('vm', 'enable_port_randomization')) + end + + def labels + Array(fetch('vm', 'labels')).map(&:to_s) + end + + def user_data + custom = custom_user_data + return custom unless custom.nil? || custom.empty? + return nil if vm_hostname.nil? + + default_hostname_cloud_init + rescue Errno::ENOENT => e + raise Error, "User data file not found: #{e.message}" + rescue Errno::EACCES => e + raise Error, "Cannot read user data file: #{e.message}" + end + + def ssh_username + fetch('ssh', 'username') + end + + def ssh_private_key_path + expand_path(fetch('ssh', 'private_key_path')) + end + + def ssh_known_hosts_path + "#{state_file}.known_hosts" + end + + def ssh_key_name + fetch('ssh', 'hyperstack_key_name') + end + + def ssh_port + Integer(fetch('ssh', 'port')) + end + + def ssh_connect_timeout + Integer(fetch('ssh', 'connect_timeout_sec')) + end + + def wireguard_udp_port + Integer(fetch('network', 'wireguard_udp_port')) + end + + def wireguard_subnet + fetch('network', 'wireguard_subnet') + end + + def ollama_port + Integer(fetch('network', 'ollama_port')) + end + + # Returns the server-side WireGuard IP for this VM. + # Uses the explicitly configured address when set; otherwise derives it as subnet_base + 1. + # Example: 192.168.3.0/24 → 192.168.3.1 (default VM1); VM2 sets wireguard_server_ip=192.168.3.3. + def wireguard_gateway_ip + configured = fetch('network', 'wireguard_server_ip') + return configured.to_s if configured && !configured.to_s.strip.empty? + + # Fall back to first usable address in the subnet. + base = IPAddr.new(wireguard_subnet).to_s + parts = base.split('.').map(&:to_i) + parts[-1] += 1 + parts.join('.') + end + + # Returns the WireGuard hostname for this VM: e.g. hyperstack1.wg1 or hyperstack2.wg1. + # Used as the DNS name to reach the VM over the tunnel (must be in /etc/hosts on the client). + def wireguard_gateway_hostname + host = vm_hostname || 'hyperstack' + "#{host}.#{local_interface_name}" + end + + def allowed_ssh_cidrs + resolved_allowed_cidrs('allowed_ssh_cidrs') + end + + def allowed_wireguard_cidrs + resolved_allowed_cidrs('allowed_wireguard_cidrs') + end + + def guest_bootstrap_enabled? + truthy?(fetch('bootstrap', 'enable_guest_bootstrap')) + end + + def install_wireguard? + truthy?(fetch('bootstrap', 'install_wireguard')) + end + + def configure_ufw? + truthy?(fetch('bootstrap', 'configure_ufw')) + end + + def configure_ollama_host? + truthy?(fetch('bootstrap', 'configure_ollama_host')) + end + + def ollama_install_enabled? + truthy?(fetch('ollama', 'install')) + end + + def ollama_models_dir + fetch('ollama', 'models_dir') + end + + def ollama_listen_host + fetch('ollama', 'listen_host') + end + + def ollama_gpu_overhead_mb + Integer(fetch('ollama', 'gpu_overhead_mb')) + end + + def ollama_num_parallel + Integer(fetch('ollama', 'num_parallel')) + end + + def ollama_context_length + Integer(fetch('ollama', 'context_length')) + end + + def ollama_pull_models + Array(fetch('ollama', 'pull_models')).map(&:to_s) + end + + def vllm_install_enabled? + truthy?(fetch('vllm', 'install')) + end + + def vllm_model + fetch('vllm', 'model') + end + + def vllm_hug_cache_dir + fetch('vllm', 'hug_cache_dir') + end + + # Derived from hug_cache_dir: sibling directory for torch.compile artifacts. + # Persisted across container restarts so recompilation is skipped on warm switches. + def vllm_compile_cache_dir + File.join(File.dirname(fetch('vllm', 'hug_cache_dir')), 'vllm_cache') + end + + def vllm_container_name + fetch('vllm', 'container_name') + end + + def vllm_max_model_len + Integer(fetch('vllm', 'max_model_len')) + end + + def vllm_gpu_memory_utilization + Float(fetch('vllm', 'gpu_memory_utilization')) + end + + def vllm_tensor_parallel_size + Integer(fetch('vllm', 'tensor_parallel_size')) + end + + def vllm_tool_call_parser + fetch('vllm', 'tool_call_parser') + end + + # Whether to pass --trust-remote-code to vLLM for the default model. + # Required for architectures not yet in the vLLM upstream registry (e.g. nemotron_h). + def vllm_trust_remote_code + truthy?(fetch('vllm', 'trust_remote_code')) + end + + # Extra vLLM CLI flags for the default model (e.g. reasoning-parser args). + def vllm_extra_args + Array(fetch('vllm', 'extra_vllm_args')).map(&:to_s) + end + + # Extra Docker -e KEY=VALUE env vars for the vLLM container (e.g. VLLM_ALLOW_LONG_MAX_MODEL_LEN=1). + def vllm_extra_docker_env + Array(fetch('vllm', 'extra_docker_env')).map(&:to_s) + end + + # Whether to pass --enable-prefix-caching to vLLM. Defaults to true. + # Disable for hybrid Mamba models (NemotronH): prefix caching forces Mamba into "all" cache + # mode which pre-allocates states for all sequences, consuming extra VRAM on startup. + def vllm_prefix_caching_enabled? + val = dig('vllm', 'enable_prefix_caching') + val.nil? || truthy?(val) + end + + def vllm_presets + Hash(dig('vllm', 'presets')).transform_keys(&:to_s) + end + + def vllm_preset_names + vllm_presets.keys + end + + def vllm_preset(name) + raw = vllm_presets[name.to_s] + unless raw + available = vllm_preset_names.empty? ? 'none configured' : vllm_preset_names.join(', ') + raise Error, "Unknown vLLM preset #{name.inspect}. Available: #{available}" + end + { + 'model' => raw['model'] || vllm_model, + 'container_name' => raw['container_name'] || vllm_container_name, + 'max_model_len' => Integer(raw['max_model_len'] || vllm_max_model_len), + 'gpu_memory_utilization' => Float(raw['gpu_memory_utilization'] || vllm_gpu_memory_utilization), + 'tensor_parallel_size' => Integer(raw['tensor_parallel_size'] || vllm_tensor_parallel_size), + 'tool_call_parser' => raw.key?('tool_call_parser') ? raw['tool_call_parser'] : vllm_tool_call_parser, + 'trust_remote_code' => raw.key?('trust_remote_code') ? raw['trust_remote_code'] : false, + 'extra_vllm_args' => raw.key?('extra_vllm_args') ? Array(raw['extra_vllm_args']) : [], + 'extra_docker_env' => raw.key?('extra_docker_env') ? Array(raw['extra_docker_env']) : [], + # nil means "not set in preset" — fall back to the top-level [vllm] value in the script. + 'enable_prefix_caching' => raw.key?('enable_prefix_caching') ? raw['enable_prefix_caching'] : nil + } + end + + def comfyui_install_enabled? + truthy?(fetch('comfyui', 'install')) + end + + def comfyui_port + Integer(fetch('comfyui', 'port')) + end + + def comfyui_models_dir + fetch('comfyui', 'models_dir') + end + + def comfyui_output_dir + fetch('comfyui', 'output_dir') + end + + def comfyui_c |
