From ee67020aa88063eccc26ce2030111d365f8cf6c2 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 25 May 2026 21:16:41 +0300 Subject: fix(inference_tester): add bounded-retry guard around vLLM chat inference Retry up to 3 times with 15/30/45s backoff on Net::ReadTimeout, Net::OpenTimeout, connection errors, non-200 HTTP, and JSON parse failures. Prevents a single transient timeout from failing the whole test run. Task referenced Manager#vllm_chat which no longer exists after refactor; applied guard to InferenceTester#chat where the call actually lives. --- lib/hyperstack/inference_tester.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'lib/hyperstack/inference_tester.rb') diff --git a/lib/hyperstack/inference_tester.rb b/lib/hyperstack/inference_tester.rb index fa8f385..e7e3aad 100644 --- a/lib/hyperstack/inference_tester.rb +++ b/lib/hyperstack/inference_tester.rb @@ -52,10 +52,23 @@ module HyperstackVM 'messages' => [{ 'role' => 'user', 'content' => prompt }], 'max_tokens' => 500 ) - resp = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 120) { |h| h.request(req) } - raise Error, "vLLM inference returned HTTP #{resp.code}" unless resp.code == '200' - JSON.parse(resp.body).dig('choices', 0, 'message', 'content').to_s.strip + retries = 3 + retries.times do |attempt| + begin + resp = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 120) { |h| h.request(req) } + raise Error, "vLLM inference returned HTTP #{resp.code}" unless resp.code == '200' + + return JSON.parse(resp.body).dig('choices', 0, 'message', 'content').to_s.strip + rescue Error, Net::ReadTimeout, Net::OpenTimeout, Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, SocketError, JSON::ParserError => e + raise Error, "vLLM inference failed after #{retries} attempts: #{e.message}" if attempt == retries - 1 + + delay = (attempt + 1) * 15 + info " vLLM inference attempt #{attempt + 1}/#{retries} failed (#{e.message}), retrying in #{delay}s..." + sleep delay + end + end end def state_vllm_enabled?(state) -- cgit v1.2.3