summaryrefslogtreecommitdiff
path: root/lib/hyperstack
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-05-25 21:16:41 +0300
committerPaul Buetow <paul@buetow.org>2026-05-25 21:16:41 +0300
commitee67020aa88063eccc26ce2030111d365f8cf6c2 (patch)
tree87e55590a771921975516762199057c526ed3357 /lib/hyperstack
parent90493d2a3d1395488f77bbedd246e09e22908c96 (diff)
fix(inference_tester): add bounded-retry guard around vLLM chat inference
Retry up to 3 times with 15/30/45s backoff on Net::ReadTimeout, Net::OpenTimeout, connection errors, non-200 HTTP, and JSON parse failures. Prevents a single transient timeout from failing the whole test run. Task referenced Manager#vllm_chat which no longer exists after refactor; applied guard to InferenceTester#chat where the call actually lives.
Diffstat (limited to 'lib/hyperstack')
-rw-r--r--lib/hyperstack/inference_tester.rb19
1 files changed, 16 insertions, 3 deletions
diff --git a/lib/hyperstack/inference_tester.rb b/lib/hyperstack/inference_tester.rb
index fa8f385..e7e3aad 100644
--- a/lib/hyperstack/inference_tester.rb
+++ b/lib/hyperstack/inference_tester.rb
@@ -52,10 +52,23 @@ module HyperstackVM
'messages' => [{ 'role' => 'user', 'content' => prompt }],
'max_tokens' => 500
)
- resp = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 120) { |h| h.request(req) }
- raise Error, "vLLM inference returned HTTP #{resp.code}" unless resp.code == '200'
- JSON.parse(resp.body).dig('choices', 0, 'message', 'content').to_s.strip
+ retries = 3
+ retries.times do |attempt|
+ begin
+ resp = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 120) { |h| h.request(req) }
+ raise Error, "vLLM inference returned HTTP #{resp.code}" unless resp.code == '200'
+
+ return JSON.parse(resp.body).dig('choices', 0, 'message', 'content').to_s.strip
+ rescue Error, Net::ReadTimeout, Net::OpenTimeout, Errno::ECONNREFUSED,
+ Errno::EHOSTUNREACH, SocketError, JSON::ParserError => e
+ raise Error, "vLLM inference failed after #{retries} attempts: #{e.message}" if attempt == retries - 1
+
+ delay = (attempt + 1) * 15
+ info " vLLM inference attempt #{attempt + 1}/#{retries} failed (#{e.message}), retrying in #{delay}s..."
+ sleep delay
+ end
+ end
end
def state_vllm_enabled?(state)