diff options
| author | Paul Buetow <paul@buetow.org> | 2026-05-25 20:05:15 +0300 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2026-05-25 20:05:15 +0300 |
| commit | 4fd86fbc75670878308fd6a56b7778334b52ccd8 (patch) | |
| tree | 1f2a611c41248844d9f7e32f4bb78760001cc8cb /hyperstack-vm1.toml | |
| parent | aa298b3d85a7f4ee7f1e1b4a3192421b95f67828 (diff) | |
refactor(manager): extract focused collaborators from Manager God Class
Extract VM lifecycle, SSH execution, WireGuard setup, model switching,
end-to-end inference tests, and provisioning orchestration into separate
collaborator classes. Manager becomes a thin facade (~80 lines).
Also fixes CLI edge-cases: status/model-list when no VMs are active,
and threads --vllm/--ollama flags through the provisioning pipeline.
Diffstat (limited to 'hyperstack-vm1.toml')
| -rw-r--r-- | hyperstack-vm1.toml | 20 |
1 files changed, 19 insertions, 1 deletions
diff --git a/hyperstack-vm1.toml b/hyperstack-vm1.toml index d28dbb8..be1107b 100644 --- a/hyperstack-vm1.toml +++ b/hyperstack-vm1.toml @@ -69,12 +69,30 @@ max_model_len = 262144 gpu_memory_utilization = 0.92 tensor_parallel_size = 1 tool_call_parser = "qwen3_coder" +extra_vllm_args = ["--reasoning-parser", "qwen3"] # Named model presets for 'ruby hyperstack.rb --vm 1 model switch <name>'. # Each preset overrides the matching [vllm] field; unset fields fall back to [vllm] defaults. +# Qwen3.6-35B-A3B AWQ — hybrid MoE (3B active / 35B total) on L40 48GB. +# Requires a quantized checkpoint (~18 GB weights) to fit alongside TurboQuant KV cache. +# Disables chunked prefill as a mitigation for vLLM issue #41726. +[vllm.presets.qwen36-35b-a3b] +model = "Qwen/Qwen3.6-35B-A3B-AWQ" +container_name = "vllm_qwen36_moe" +max_model_len = 65536 +gpu_memory_utilization = 0.92 +tensor_parallel_size = 1 +tool_call_parser = "qwen3_coder" +docker_image = "vllm/vllm-openai:nightly" +extra_vllm_args = [ + "--reasoning-parser", "qwen3", + "--kv-cache-dtype", "turboquant_k8v4", + "--no-enable-chunked-prefill" +] + # Qwen3.6-27B FP8 — dense 27B multimodal model with native 262K context. -# Uses qwen3 reasoning parsing plus qwen3_coder tool calling on vLLM >=0.19.0. +# A100/H100 fallback preset; uses qwen3 reasoning parsing plus qwen3_coder tool calling. [vllm.presets.qwen36-27b] model = "Qwen/Qwen3.6-27B-FP8" container_name = "vllm_qwen36_27b" |
