project flow diagram

5 files changed, 327 insertions, 18 deletions

diff --git a/lib/dsl.rb b/lib/dsl.rb
index 0414050..ff147f3 100644
--- a/lib/dsl.rb
+++ b/lib/dsl.rb
@@ -7,6 +7,7 @@ require_relative 'log'
 require_relative 'chained'
 
 require_relative 'dslkeywords/agent'
+require_relative 'dslkeywords/command'
 require_relative 'dslkeywords/prompt'
 require_relative 'dslkeywords/file'
 require_relative 'dslkeywords/symlink'
@@ -38,6 +39,7 @@ module RCM
     class DuplicateResource < StandardError; end
     class DuplicateDefinition < StandardError; end
     class NoSuchAgentDefinition < StandardError; end
+    class NoSuchCommandDefinition < StandardError; end
     class NoSuchPromptDefinition < StandardError; end
 
     def initialize(reset)
diff --git a/lib/dslkeywords/agent.rb b/lib/dslkeywords/agent.rb
index 6632835..3d6874c 100644
--- a/lib/dslkeywords/agent.rb
+++ b/lib/dslkeywords/agent.rb
@@ -8,6 +8,7 @@ module RCM
     attr_reader :name
 
     class InvalidName < StandardError; end
+    class InvalidRetrySetting < StandardError; end
 
     def self.id_for(name) = super(normalize_name(name))
 
@@ -20,6 +21,9 @@ module RCM
 
     def initialize(name)
       @name = self.class.normalize_name(name)
+      @retries = 2
+      @retry_delay = 1.0
+      @retry_backoff = 2.0
       super(@name)
     end
 
@@ -28,6 +32,39 @@ module RCM
 
       @command = text.to_s
     end
+
+    def retries(value = nil)
+      return @retries if value.nil?
+
+      @retries = Integer(value)
+      raise InvalidRetrySetting, 'Retry count must be non-negative' if @retries.negative?
+
+      @retries
+    rescue ArgumentError, TypeError
+      raise InvalidRetrySetting, "Invalid retry count: #{value.inspect}"
+    end
+
+    def retry_delay(value = nil)
+      return @retry_delay if value.nil?
+
+      @retry_delay = Float(value)
+      raise InvalidRetrySetting, 'Retry delay must be non-negative' if @retry_delay.negative?
+
+      @retry_delay
+    rescue ArgumentError, TypeError
+      raise InvalidRetrySetting, "Invalid retry delay: #{value.inspect}"
+    end
+
+    def retry_backoff(value = nil)
+      return @retry_backoff if value.nil?
+
+      @retry_backoff = Float(value)
+      raise InvalidRetrySetting, 'Retry backoff must be at least 1.0' if @retry_backoff < 1.0
+
+      @retry_backoff
+    rescue ArgumentError, TypeError
+      raise InvalidRetrySetting, "Invalid retry backoff: #{value.inspect}"
+    end
   end
 
   # Adds the `agent` definition keyword to the top-level DSL.
diff --git a/lib/dslkeywords/file.rb b/lib/dslkeywords/file.rb
index 1e40691..06bb5ce 100644
--- a/lib/dslkeywords/file.rb
+++ b/lib/dslkeywords/file.rb
@@ -3,12 +3,14 @@
 require 'digest'
 require 'erb'
 require 'fileutils'
+require 'json'
 require 'open3'
 require 'shellwords'
 require 'tempfile'
 
 require_relative 'resource'
 require_relative '../chained'
+require_relative 'command'
 require_relative 'file_backup'
 
 module RCM
@@ -131,7 +133,11 @@ module RCM
   # sourcefile reading. Touch and Directory extend BasicFile directly so
   # they are not burdened with content/from (ISP).
   class BaseFile < BasicFile
-    def from(what) = @from = validate(__method__, what.to_sym, :sourcefile, :template)
+    def from(what)
+      return what if what.is_a?(CommandDefinition::Reference)
+
+      @from = validate(__method__, what.to_sym, :sourcefile, :template)
+    end
 
     # Return or set the resource's content.
     # Getter: resolves ERB templates or reads sourcefile on demand.
@@ -149,12 +155,22 @@ module RCM
   # delete. Writes via a temp file so the final rename is atomic.
   # rubocop:disable Metrics/ClassLength
   class File < BaseFile
+    include CommandComposable
+
     class AgentCommandFailed < StandardError; end
+    class CommandFailed < StandardError; end
+    class InvalidComposition < StandardError; end
     class InvalidAgentSpec < StandardError; end
+    class InvalidAgentCacheRecord < StandardError; end
     class MissingAgentInput < StandardError; end
 
     attr_reader :agent_name, :prompt_name
 
+    def initialize(file_path)
+      super(file_path)
+      initialize_command_composition!
+    end
+
     def agent(spec = nil, prompt_name = nil)
       agent_name, prompt_name = resolved_agent_spec(spec, prompt_name)
 
@@ -170,6 +186,12 @@ module RCM
 
     def line(line) = @ensure_line = line
 
+    def content(text = nil)
+      return render_composed_content(super(), @file_path) if text.nil?
+
+      super
+    end
+
     def evaluate!
       return unless super
 
@@ -254,16 +276,23 @@ module RCM
       raise MissingAgentInput, "File #{@file_path} does not exist for agent processing" unless ::File.file?(@file_path)
 
       agent_definition, prompt_definition = agent_configuration!
+      cache_state = agent_cache_state(agent_definition, prompt_definition)
+      return skip_fresh_agent_processing! unless cache_state[:stale]
 
-      if option :dry
-        info "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name} - dry run!"
-        return
-      end
+      info "Agent cache is stale for #{@file_path}: #{cache_state[:reason]}"
+      process_stale_agent!(agent_definition, prompt_definition, cache_state)
+    rescue AgentCommandFailed => e
+      warn "#{e.message}. Skipping #{@file_path} and continuing"
+    end
 
+    def process_stale_agent!(agent_definition, prompt_definition, cache_state)
       input = ::File.read(@file_path)
       output = run_agent!(input, agent_definition, prompt_definition)
+      return if option :dry
+
       create_parent_directory! unless ::File.directory?(::File.dirname(@file_path))
       write!(output)
+      refresh_agent_cache_record!(cache_state[:record_path], cache_state[:definition_fingerprint])
     end
 
     # rubocop:disable Metrics/MethodLength
@@ -294,25 +323,121 @@ module RCM
     end
     # rubocop:enable Metrics/MethodLength
 
-    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     def run_agent!(input, agent_definition, prompt_definition)
+      with_agent_input_file(input) do |input_path|
+        prompt_text = prompt_definition.render(@file_path)
+        return dry_run_agent_input(input) if option :dry
+
+        command = render_agent_command(agent_definition.command.to_s, prompt_text, input_path)
+        execute_agent_command_with_retries(command, input, agent_definition)
+      end
+    end
+
+    def execute_agent_command_with_retries(command, input, agent_definition)
+      attempt = 1
+      retry_state = agent_retry_state(agent_definition)
+
+      loop do
+        stdout, stderr, status = capture_agent_command(command, input, attempt, retry_state[:max_attempts])
+        return stdout if status.success?
+
+        retry_state[:delay] = handle_failed_agent_command(status, stderr, attempt, retry_state)
+        attempt += 1
+      end
+    end
+
+    def agent_retry_state(agent_definition)
+      {
+        max_attempts: agent_definition.retries + 1,
+        delay: agent_definition.retry_delay,
+        backoff: agent_definition.retry_backoff
+      }
+    end
+
+    def handle_failed_agent_command(status, stderr, attempt, retry_state)
+      failure_message = agent_command_failure_message(status, stderr, attempt, retry_state[:max_attempts])
+      retry_agent_command!(
+        failure_message,
+        attempt,
+        retry_state[:max_attempts],
+        retry_state[:delay],
+        retry_state[:backoff]
+      )
+    end
+
+    def with_agent_input_file(input)
       Tempfile.create(['rcm-agent-input', '.txt']) do |tmp|
         tmp.write(input)
         tmp.flush
         tmp.close
+        yield tmp.path
+      end
+    end
 
-        command = render_agent_command(agent_definition.command.to_s, prompt_definition.text.to_s, tmp.path)
-        info "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name}"
-        stdout, stderr, status = Open3.capture3(command, stdin_data: input)
-        return stdout if status.success?
+    def dry_run_agent_input(input)
+      info "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name} - dry run!"
+      input
+    end
+
+    def capture_agent_command(command, input, attempt, max_attempts)
+      info agent_processing_message(attempt, max_attempts)
+      Open3.popen3(command) do |stdin, stdout, stderr, wait_thread|
+        stdin.write(input)
+        stdin.close
+
+        stdout_buffer, stderr_buffer, stream_threads = start_agent_streams(stdout, stderr)
+        stream_threads.each(&:join)
+
+        [stdout_buffer, stderr_buffer, wait_thread.value]
+      end
+    end
+
+    def retry_agent_command!(failure_message, attempt, max_attempts, retry_delay, retry_backoff)
+      raise AgentCommandFailed, failure_message if attempt >= max_attempts
 
-        message = stderr.to_s.strip
-        message = 'no stderr output' if message.empty?
-        raise AgentCommandFailed,
-              "Agent #{@agent_name} failed for #{@file_path} (exit #{status.exitstatus}): #{message}"
+      warn "#{failure_message}. Retrying in #{formatted_retry_delay(retry_delay)}s"
+      sleep(retry_delay) if retry_delay.positive?
+      retry_delay * retry_backoff
+    end
+
+    def stream_agent_io(source, buffer, destination)
+      loop do
+        chunk = source.readpartial(4096)
+        buffer << chunk
+        destination.write(chunk)
+        destination.flush
+      rescue EOFError
+        break
       end
+
+      destination.puts unless buffer.empty? || buffer.end_with?("\n")
+    end
+
+    def start_agent_streams(stdout, stderr)
+      stdout_buffer = +''
+      stderr_buffer = +''
+      stream_threads = [
+        Thread.new { stream_agent_io(stdout, stdout_buffer, $stdout) },
+        Thread.new { stream_agent_io(stderr, stderr_buffer, $stderr) }
+      ]
+      [stdout_buffer, stderr_buffer, stream_threads]
+    end
+
+    def agent_processing_message(attempt, max_attempts)
+      suffix = max_attempts > 1 ? " (attempt #{attempt}/#{max_attempts})" : ''
+      "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name}#{suffix}"
+    end
+
+    def agent_command_failure_message(status, stderr, attempt, max_attempts)
+      message = stderr.to_s.strip
+      message = 'no stderr output' if message.empty?
+      attempt_suffix = max_attempts > 1 ? ", attempt #{attempt}/#{max_attempts}" : ''
+      "Agent #{@agent_name} failed for #{@file_path} (exit #{status.exitstatus}#{attempt_suffix}): #{message}"
+    end
+
+    def formatted_retry_delay(retry_delay)
+      format('%.3g', retry_delay)
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def agent_configuration!
       [
@@ -328,6 +453,106 @@ module RCM
       command.gsub!(/\bFILE_PATH\b/, Shellwords.escape(@file_path))
       command
     end
+
+    def agent_cache_state(agent_definition, prompt_definition)
+      file_checksum = checksum_for(@file_path)
+      definition_fingerprint = agent_definition_fingerprint(agent_definition, prompt_definition)
+      record_path = agent_cache_record_path
+      stale, reason = stale_agent_cache_record?(
+        load_agent_cache_record(record_path),
+        file_checksum,
+        definition_fingerprint
+      )
+
+      { stale:, reason:, record_path:, definition_fingerprint: }
+    end
+
+    def stale_agent_cache_record?(record, file_checksum, definition_fingerprint)
+      return [true, 'no track record'] if record.nil?
+      return [true, 'cache format changed'] if record['version'] != 2 || record['file_checksum'].nil?
+      return [true, 'file checksum changed'] if record['file_checksum'] != file_checksum
+      return [true, 'agent or prompt changed'] if record['definition_fingerprint'] != definition_fingerprint
+
+      [false, 'fresh']
+    end
+
+    def agent_definition_fingerprint(agent_definition, prompt_definition)
+      Digest::SHA256.hexdigest(
+        JSON.generate(
+          agent_command: agent_definition.command.to_s,
+          prompt: prompt_definition.fingerprint_source
+        )
+      )
+    end
+
+    def refresh_agent_cache_record!(record_path, definition_fingerprint)
+      persist_agent_cache_record!(
+        record_path,
+        agent_cache_record_payload(checksum_for(@file_path), definition_fingerprint)
+      )
+    end
+
+    def agent_cache_record_payload(file_checksum, definition_fingerprint)
+      {
+        version: 2,
+        file_path: expanded_file_path,
+        agent_name: @agent_name,
+        prompt_name: @prompt_name,
+        file_checksum:,
+        definition_fingerprint:,
+        recorded_at_ns: current_time_ns
+      }
+    end
+
+    def persist_agent_cache_record!(record_path, payload)
+      cache_dir = ::File.dirname(record_path)
+      ::FileUtils.mkdir_p(cache_dir)
+      tmp_path = "#{record_path}.tmp"
+      ::File.write(tmp_path, JSON.generate(payload))
+      ::File.rename(tmp_path, record_path)
+    rescue SystemCallError => e
+      warn "Unable to persist agent cache #{record_path}: #{e.message}"
+      ::File.delete(tmp_path) if defined?(tmp_path) && ::File.file?(tmp_path)
+    end
+
+    def load_agent_cache_record(record_path)
+      return unless ::File.file?(record_path)
+
+      JSON.parse(::File.read(record_path))
+    rescue JSON::ParserError, SystemCallError => e
+      warn "Ignoring invalid agent cache #{record_path}: #{e.message}"
+      nil
+    end
+
+    def agent_cache_record_path
+      key = Digest::SHA256.hexdigest(
+        JSON.generate(file_path: expanded_file_path, agent_name: @agent_name, prompt_name: @prompt_name)
+      )
+      ::File.join(agent_cache_dir, "#{key}.json")
+    end
+
+    def agent_cache_dir
+      cache_root = ENV['XDG_CACHE_HOME'] || ::File.expand_path('~/.cache')
+      ::File.join(cache_root, 'rcm', 'agents')
+    end
+
+    def expanded_file_path = ::File.expand_path(@file_path)
+
+    def current_time_ns
+      now = ::Time.now
+      (now.to_i * 1_000_000_000) + now.nsec
+    end
+
+    def skip_fresh_agent_processing!
+      info "Skipping #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name}; agent cache is fresh"
+    end
+
+    def invalid_command_composition!(position)
+      raise InvalidComposition, "#{position} expects `from command ...`"
+    end
+
+    def command_failure_class = CommandFailed
+    def command_composition_subject = "file #{@file_path}"
   end
   # rubocop:enable Metrics/ClassLength
 
diff --git a/lib/dslkeywords/file_backup.rb b/lib/dslkeywords/file_backup.rb
index 210804c..1716aa5 100644
--- a/lib/dslkeywords/file_backup.rb
+++ b/lib/dslkeywords/file_backup.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest'
 
 module RCM
@@ -5,12 +7,16 @@ module RCM
   # Included by BasicFile so all file/directory/symlink resources share
   # the same backup logic.
   module FileBackup
+    def checksum_for(file_path)
+      Digest::SHA256.file(file_path).hexdigest
+    end
+
     # TODO: Make protected?
     def backup!(file_path, checksum = nil)
       return if @without_backup
 
       suffix = if ::File.file?(file_path)
-                 checksum.nil? ? Digest::SHA256.file(file_path).hexdigest : checksum
+                 checksum.nil? ? checksum_for(file_path) : checksum
                else
                  Time.now.strftime('%s-%L')
                end
@@ -18,8 +24,8 @@ module RCM
     end
 
     def different?(file_a, file_b)
-      checksum_a = Digest::SHA256.file(file_a).hexdigest
-      checksum_b = Digest::SHA256.file(file_b).hexdigest
+      checksum_a = checksum_for(file_a)
+      checksum_b = checksum_for(file_b)
       [checksum_a != checksum_b, checksum_a, checksum_b]
     end
 
diff --git a/lib/dslkeywords/prompt.rb b/lib/dslkeywords/prompt.rb
index 9e599ef..1b383b9 100644
--- a/lib/dslkeywords/prompt.rb
+++ b/lib/dslkeywords/prompt.rb
@@ -1,13 +1,20 @@
 # frozen_string_literal: true
 
+require_relative '../chained'
+require_relative 'command'
 require_relative 'keyword'
 
 module RCM
   # Stores a named prompt body for agent-backed file processing.
   class PromptDefinition < Keyword
+    include Chained
+    include CommandComposable
+
     attr_reader :name
 
     class InvalidName < StandardError; end
+    class InvalidComposition < StandardError; end
+    class CommandFailed < StandardError; end
 
     def self.id_for(name) = super(normalize_name(name))
 
@@ -20,6 +27,7 @@ module RCM
 
     def initialize(name)
       @name = self.class.normalize_name(name)
+      initialize_command_composition!
       super(@name)
     end
 
@@ -28,6 +36,37 @@ module RCM
 
       @text = value.to_s
     end
+
+    def from(value = nil) = value
+
+    def render(file_path) = render_composed_content(text.to_s, file_path)
+
+    def fingerprint_source
+      {
+        text: text.to_s,
+        composition: @command_composition.map { |entry| fingerprinted_command(entry) }
+      }
+    end
+
+    private
+
+    def fingerprinted_command(entry)
+      command_definition = dsl.object!(
+        CommandDefinition,
+        entry[:name],
+        error_class: DSL::NoSuchCommandDefinition,
+        kind: 'command'
+      )
+
+      entry.merge(template: command_definition.template.to_s)
+    end
+
+    def invalid_command_composition!(position)
+      raise InvalidComposition, "#{position} expects `from command ...`"
+    end
+
+    def command_failure_class = CommandFailed
+    def command_composition_subject = "prompt #{@name}"
   end
 
   # Adds the `prompt` definition keyword to the top-level DSL.