6 files changed, 502 insertions, 21 deletions

diff --git a/lib/dsl.rb b/lib/dsl.rb
index 0414050..ff147f3 100644
--- a/lib/dsl.rb
+++ b/lib/dsl.rb
@@ -7,6 +7,7 @@ require_relative 'log'
 require_relative 'chained'
 
 require_relative 'dslkeywords/agent'
+require_relative 'dslkeywords/command'
 require_relative 'dslkeywords/prompt'
 require_relative 'dslkeywords/file'
 require_relative 'dslkeywords/symlink'
@@ -38,6 +39,7 @@ module RCM
     class DuplicateResource < StandardError; end
     class DuplicateDefinition < StandardError; end
     class NoSuchAgentDefinition < StandardError; end
+    class NoSuchCommandDefinition < StandardError; end
     class NoSuchPromptDefinition < StandardError; end
 
     def initialize(reset)
diff --git a/lib/dslkeywords/agent.rb b/lib/dslkeywords/agent.rb
index 6632835..3d6874c 100644
--- a/lib/dslkeywords/agent.rb
+++ b/lib/dslkeywords/agent.rb
@@ -8,6 +8,7 @@ module RCM
     attr_reader :name
 
     class InvalidName < StandardError; end
+    class InvalidRetrySetting < StandardError; end
 
     def self.id_for(name) = super(normalize_name(name))
 
@@ -20,6 +21,9 @@ module RCM
 
     def initialize(name)
       @name = self.class.normalize_name(name)
+      @retries = 2
+      @retry_delay = 1.0
+      @retry_backoff = 2.0
       super(@name)
     end
 
@@ -28,6 +32,39 @@ module RCM
 
       @command = text.to_s
     end
+
+    def retries(value = nil)
+      return @retries if value.nil?
+
+      @retries = Integer(value)
+      raise InvalidRetrySetting, 'Retry count must be non-negative' if @retries.negative?
+
+      @retries
+    rescue ArgumentError, TypeError
+      raise InvalidRetrySetting, "Invalid retry count: #{value.inspect}"
+    end
+
+    def retry_delay(value = nil)
+      return @retry_delay if value.nil?
+
+      @retry_delay = Float(value)
+      raise InvalidRetrySetting, 'Retry delay must be non-negative' if @retry_delay.negative?
+
+      @retry_delay
+    rescue ArgumentError, TypeError
+      raise InvalidRetrySetting, "Invalid retry delay: #{value.inspect}"
+    end
+
+    def retry_backoff(value = nil)
+      return @retry_backoff if value.nil?
+
+      @retry_backoff = Float(value)
+      raise InvalidRetrySetting, 'Retry backoff must be at least 1.0' if @retry_backoff < 1.0
+
+      @retry_backoff
+    rescue ArgumentError, TypeError
+      raise InvalidRetrySetting, "Invalid retry backoff: #{value.inspect}"
+    end
   end
 
   # Adds the `agent` definition keyword to the top-level DSL.
diff --git a/lib/dslkeywords/command.rb b/lib/dslkeywords/command.rb
new file mode 100644
index 0000000..54a7f4b
--- /dev/null
+++ b/lib/dslkeywords/command.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'shellwords'
+
+require_relative 'keyword'
+
+module RCM
+  # Stores a named shell command template for prompt-time expansion.
+  class CommandDefinition < Keyword
+    Reference = Struct.new(:name, keyword_init: true)
+
+    attr_reader :name
+
+    class InvalidName < StandardError; end
+
+    def self.id_for(name) = super(normalize_name(name))
+    def self.reference(name) = Reference.new(name: normalize_name(name))
+
+    def self.normalize_name(name)
+      normalized = name.to_s.strip.gsub(/\s+/, ' ')
+      raise InvalidName, 'Command name must not be empty' if normalized.empty?
+
+      normalized
+    end
+
+    def initialize(name)
+      @name = self.class.normalize_name(name)
+      super(@name)
+    end
+
+    def template(text = nil)
+      return @template if text.nil?
+
+      @template = text.to_s
+    end
+  end
+
+  # Shared append/prepend-from-command behavior for prompt and file content.
+  module CommandComposable
+    def initialize_command_composition!
+      @command_composition = []
+    end
+
+    def append(value = nil)
+      compose_command(:append, value)
+      nil
+    end
+
+    def prepend(value = nil)
+      compose_command(:prepend, value)
+      nil
+    end
+
+    def command(name = nil)
+      CommandDefinition.reference(name)
+    end
+
+    private
+
+    def render_composed_content(base_text, file_path)
+      prepended = render_command_sections(:prepend, file_path)
+      appended = render_command_sections(:append, file_path)
+      join_command_sections(*prepended, base_text.to_s, *appended)
+    end
+
+    def compose_command(position, value)
+      invalid_command_composition!(position) unless value.is_a?(CommandDefinition::Reference)
+
+      @command_composition << { position:, name: value.name }
+    end
+
+    def render_command_sections(position, file_path)
+      @command_composition
+        .select { |entry| entry[:position] == position }
+        .map { |entry| execute_composed_command(entry[:name], file_path) }
+    end
+
+    def execute_composed_command(name, file_path)
+      command = resolved_command(name, file_path)
+      return dry_run_command(name, file_path) if option :dry
+
+      stdout, stderr, status = Open3.capture3(command)
+      return stdout if status.success?
+
+      raise command_failure_class, command_failure_message(name, file_path, status.exitstatus, stderr)
+    end
+
+    def resolved_command(name, file_path)
+      command_definition = dsl.object!(
+        CommandDefinition,
+        name,
+        error_class: DSL::NoSuchCommandDefinition,
+        kind: 'command'
+      )
+
+      render_command_template(command_definition.template.to_s, file_path)
+    end
+
+    def render_command_template(template, file_path)
+      command = template.dup
+      command.gsub!(/\bFILE_PATH\b/, Shellwords.escape(file_path))
+      command
+    end
+
+    def dry_run_command(name, file_path)
+      info "Running command #{name} for #{command_composition_subject} on #{file_path} - dry run!"
+      ''
+    end
+
+    def command_failure_message(name, file_path, exit_status, stderr)
+      message = stderr.to_s.strip
+      message = 'no stderr output' if message.empty?
+      "Command #{name} failed while rendering #{command_composition_subject} for #{file_path} " \
+      "(exit #{exit_status}): #{message}"
+    end
+
+    def join_command_sections(*sections)
+      sections.each_with_object(+'') do |section, result|
+        next if section.nil? || section.empty?
+
+        result << "\n" if !result.empty? && !result.end_with?("\n") && !section.start_with?("\n")
+        result << section
+      end
+    end
+  end
+
+  # Adds the `command` definition keyword to the top-level DSL.
+  class DSL
+    def command(name = nil, &block)
+      return name if name.nil?
+      return unless @conds_met
+
+      definition = CommandDefinition.new(name)
+      definition.dsl = self
+      definition.template(definition.instance_eval(&block)) if block
+      register(definition, schedule: false, duplicate_error: DuplicateDefinition)
+    end
+  end
+end
diff --git a/lib/dslkeywords/file.rb b/lib/dslkeywords/file.rb
index 08f8f48..06bb5ce 100644
--- a/lib/dslkeywords/file.rb
+++ b/lib/dslkeywords/file.rb
@@ -3,12 +3,14 @@
 require 'digest'
 require 'erb'
 require 'fileutils'
+require 'json'
 require 'open3'
 require 'shellwords'
 require 'tempfile'
 
 require_relative 'resource'
 require_relative '../chained'
+require_relative 'command'
 require_relative 'file_backup'
 
 module RCM
@@ -131,7 +133,11 @@ module RCM
   # sourcefile reading. Touch and Directory extend BasicFile directly so
   # they are not burdened with content/from (ISP).
   class BaseFile < BasicFile
-    def from(what) = @from = validate(__method__, what.to_sym, :sourcefile, :template)
+    def from(what)
+      return what if what.is_a?(CommandDefinition::Reference)
+
+      @from = validate(__method__, what.to_sym, :sourcefile, :template)
+    end
 
     # Return or set the resource's content.
     # Getter: resolves ERB templates or reads sourcefile on demand.
@@ -149,16 +155,24 @@ module RCM
   # delete. Writes via a temp file so the final rename is atomic.
   # rubocop:disable Metrics/ClassLength
   class File < BaseFile
+    include CommandComposable
+
     class AgentCommandFailed < StandardError; end
+    class CommandFailed < StandardError; end
+    class InvalidComposition < StandardError; end
     class InvalidAgentSpec < StandardError; end
+    class InvalidAgentCacheRecord < StandardError; end
     class MissingAgentInput < StandardError; end
 
     attr_reader :agent_name, :prompt_name
 
+    def initialize(file_path)
+      super(file_path)
+      initialize_command_composition!
+    end
+
     def agent(spec = nil, prompt_name = nil)
-      agent_name = normalize_agent_reference(spec)
-      prompt_name = normalize_agent_reference(prompt_name)
-      agent_name, prompt_name = agent_name.split(/\s+/, 2) if prompt_name.nil? && agent_name&.include?(' ')
+      agent_name, prompt_name = resolved_agent_spec(spec, prompt_name)
 
       if agent_name.nil? || prompt_name.nil?
         raise InvalidAgentSpec, 'Expected exactly one agent name and one prompt name'
@@ -172,6 +186,12 @@ module RCM
 
     def line(line) = @ensure_line = line
 
+    def content(text = nil)
+      return render_composed_content(super(), @file_path) if text.nil?
+
+      super
+    end
+
     def evaluate!
       return unless super
 
@@ -218,20 +238,61 @@ module RCM
       normalized.gsub(/\s+/, ' ')
     end
 
+    def resolved_agent_spec(spec, prompt_name)
+      agent_name = normalize_agent_reference(spec)
+      prompt_name = normalize_agent_reference(prompt_name)
+      candidates = resolved_agent_candidates(agent_name, prompt_name)
+
+      return candidates.first if candidates.one?
+      raise InvalidAgentSpec, 'Ambiguous agent specification' if candidates.length > 1
+      return [agent_name, prompt_name] unless prompt_name.nil?
+      return [agent_name, nil] unless agent_name&.include?(' ')
+
+      agent_name.split(/\s+/, 2)
+    end
+
+    def resolved_agent_candidates(agent_name, prompt_name)
+      phrase = [agent_name, prompt_name].compact.join(' ')
+      parts = phrase.split(/\s+/)
+      return [] if parts.length < 2
+
+      (1...parts.length).filter_map do |index|
+        candidate_agent_name = parts[0...index].join(' ')
+        candidate_prompt_name = parts[index..].join(' ')
+        next unless definition_registered?(AgentDefinition, candidate_agent_name)
+        next unless definition_registered?(PromptDefinition, candidate_prompt_name)
+
+        [candidate_agent_name, candidate_prompt_name]
+      end
+    end
+
+    def definition_registered?(klass, name)
+      dsl.class.object(klass.id_for(name))
+    rescue klass::InvalidName
+      false
+    end
+
     def evaluate_agent_processing!
       raise MissingAgentInput, "File #{@file_path} does not exist for agent processing" unless ::File.file?(@file_path)
 
       agent_definition, prompt_definition = agent_configuration!
+      cache_state = agent_cache_state(agent_definition, prompt_definition)
+      return skip_fresh_agent_processing! unless cache_state[:stale]
 
-      if option :dry
-        info "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name} - dry run!"
-        return
-      end
+      info "Agent cache is stale for #{@file_path}: #{cache_state[:reason]}"
+      process_stale_agent!(agent_definition, prompt_definition, cache_state)
+    rescue AgentCommandFailed => e
+      warn "#{e.message}. Skipping #{@file_path} and continuing"
+    end
 
+    def process_stale_agent!(agent_definition, prompt_definition, cache_state)
       input = ::File.read(@file_path)
       output = run_agent!(input, agent_definition, prompt_definition)
+      return if option :dry
+
       create_parent_directory! unless ::File.directory?(::File.dirname(@file_path))
       write!(output)
+      refresh_agent_cache_record!(cache_state[:record_path], cache_state[:definition_fingerprint])
     end
 
     # rubocop:disable Metrics/MethodLength
@@ -262,25 +323,121 @@ module RCM
     end
     # rubocop:enable Metrics/MethodLength
 
-    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     def run_agent!(input, agent_definition, prompt_definition)
+      with_agent_input_file(input) do |input_path|
+        prompt_text = prompt_definition.render(@file_path)
+        return dry_run_agent_input(input) if option :dry
+
+        command = render_agent_command(agent_definition.command.to_s, prompt_text, input_path)
+        execute_agent_command_with_retries(command, input, agent_definition)
+      end
+    end
+
+    def execute_agent_command_with_retries(command, input, agent_definition)
+      attempt = 1
+      retry_state = agent_retry_state(agent_definition)
+
+      loop do
+        stdout, stderr, status = capture_agent_command(command, input, attempt, retry_state[:max_attempts])
+        return stdout if status.success?
+
+        retry_state[:delay] = handle_failed_agent_command(status, stderr, attempt, retry_state)
+        attempt += 1
+      end
+    end
+
+    def agent_retry_state(agent_definition)
+      {
+        max_attempts: agent_definition.retries + 1,
+        delay: agent_definition.retry_delay,
+        backoff: agent_definition.retry_backoff
+      }
+    end
+
+    def handle_failed_agent_command(status, stderr, attempt, retry_state)
+      failure_message = agent_command_failure_message(status, stderr, attempt, retry_state[:max_attempts])
+      retry_agent_command!(
+        failure_message,
+        attempt,
+        retry_state[:max_attempts],
+        retry_state[:delay],
+        retry_state[:backoff]
+      )
+    end
+
+    def with_agent_input_file(input)
       Tempfile.create(['rcm-agent-input', '.txt']) do |tmp|
         tmp.write(input)
         tmp.flush
         tmp.close
+        yield tmp.path
+      end
+    end
 
-        command = render_agent_command(agent_definition.command.to_s, prompt_definition.text.to_s, tmp.path)
-        info "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name}"
-        stdout, stderr, status = Open3.capture3(command, stdin_data: input)
-        return stdout if status.success?
+    def dry_run_agent_input(input)
+      info "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name} - dry run!"
+      input
+    end
+
+    def capture_agent_command(command, input, attempt, max_attempts)
+      info agent_processing_message(attempt, max_attempts)
+      Open3.popen3(command) do |stdin, stdout, stderr, wait_thread|
+        stdin.write(input)
+        stdin.close
+
+        stdout_buffer, stderr_buffer, stream_threads = start_agent_streams(stdout, stderr)
+        stream_threads.each(&:join)
+
+        [stdout_buffer, stderr_buffer, wait_thread.value]
+      end
+    end
 
-        message = stderr.to_s.strip
-        message = 'no stderr output' if message.empty?
-        raise AgentCommandFailed,
-              "Agent #{@agent_name} failed for #{@file_path} (exit #{status.exitstatus}): #{message}"
+    def retry_agent_command!(failure_message, attempt, max_attempts, retry_delay, retry_backoff)
+      raise AgentCommandFailed, failure_message if attempt >= max_attempts
+
+      warn "#{failure_message}. Retrying in #{formatted_retry_delay(retry_delay)}s"
+      sleep(retry_delay) if retry_delay.positive?
+      retry_delay * retry_backoff
+    end
+
+    def stream_agent_io(source, buffer, destination)
+      loop do
+        chunk = source.readpartial(4096)
+        buffer << chunk
+        destination.write(chunk)
+        destination.flush
+      rescue EOFError
+        break
       end
+
+      destination.puts unless buffer.empty? || buffer.end_with?("\n")
+    end
+
+    def start_agent_streams(stdout, stderr)
+      stdout_buffer = +''
+      stderr_buffer = +''
+      stream_threads = [
+        Thread.new { stream_agent_io(stdout, stdout_buffer, $stdout) },
+        Thread.new { stream_agent_io(stderr, stderr_buffer, $stderr) }
+      ]
+      [stdout_buffer, stderr_buffer, stream_threads]
+    end
+
+    def agent_processing_message(attempt, max_attempts)
+      suffix = max_attempts > 1 ? " (attempt #{attempt}/#{max_attempts})" : ''
+      "Processing #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name}#{suffix}"
+    end
+
+    def agent_command_failure_message(status, stderr, attempt, max_attempts)
+      message = stderr.to_s.strip
+      message = 'no stderr output' if message.empty?
+      attempt_suffix = max_attempts > 1 ? ", attempt #{attempt}/#{max_attempts}" : ''
+      "Agent #{@agent_name} failed for #{@file_path} (exit #{status.exitstatus}#{attempt_suffix}): #{message}"
+    end
+
+    def formatted_retry_delay(retry_delay)
+      format('%.3g', retry_delay)
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def agent_configuration!
       [
@@ -296,6 +453,106 @@ module RCM
       command.gsub!(/\bFILE_PATH\b/, Shellwords.escape(@file_path))
       command
     end
+
+    def agent_cache_state(agent_definition, prompt_definition)
+      file_checksum = checksum_for(@file_path)
+      definition_fingerprint = agent_definition_fingerprint(agent_definition, prompt_definition)
+      record_path = agent_cache_record_path
+      stale, reason = stale_agent_cache_record?(
+        load_agent_cache_record(record_path),
+        file_checksum,
+        definition_fingerprint
+      )
+
+      { stale:, reason:, record_path:, definition_fingerprint: }
+    end
+
+    def stale_agent_cache_record?(record, file_checksum, definition_fingerprint)
+      return [true, 'no track record'] if record.nil?
+      return [true, 'cache format changed'] if record['version'] != 2 || record['file_checksum'].nil?
+      return [true, 'file checksum changed'] if record['file_checksum'] != file_checksum
+      return [true, 'agent or prompt changed'] if record['definition_fingerprint'] != definition_fingerprint
+
+      [false, 'fresh']
+    end
+
+    def agent_definition_fingerprint(agent_definition, prompt_definition)
+      Digest::SHA256.hexdigest(
+        JSON.generate(
+          agent_command: agent_definition.command.to_s,
+          prompt: prompt_definition.fingerprint_source
+        )
+      )
+    end
+
+    def refresh_agent_cache_record!(record_path, definition_fingerprint)
+      persist_agent_cache_record!(
+        record_path,
+        agent_cache_record_payload(checksum_for(@file_path), definition_fingerprint)
+      )
+    end
+
+    def agent_cache_record_payload(file_checksum, definition_fingerprint)
+      {
+        version: 2,
+        file_path: expanded_file_path,
+        agent_name: @agent_name,
+        prompt_name: @prompt_name,
+        file_checksum:,
+        definition_fingerprint:,
+        recorded_at_ns: current_time_ns
+      }
+    end
+
+    def persist_agent_cache_record!(record_path, payload)
+      cache_dir = ::File.dirname(record_path)
+      ::FileUtils.mkdir_p(cache_dir)
+      tmp_path = "#{record_path}.tmp"
+      ::File.write(tmp_path, JSON.generate(payload))
+      ::File.rename(tmp_path, record_path)
+    rescue SystemCallError => e
+      warn "Unable to persist agent cache #{record_path}: #{e.message}"
+      ::File.delete(tmp_path) if defined?(tmp_path) && ::File.file?(tmp_path)
+    end
+
+    def load_agent_cache_record(record_path)
+      return unless ::File.file?(record_path)
+
+      JSON.parse(::File.read(record_path))
+    rescue JSON::ParserError, SystemCallError => e
+      warn "Ignoring invalid agent cache #{record_path}: #{e.message}"
+      nil
+    end
+
+    def agent_cache_record_path
+      key = Digest::SHA256.hexdigest(
+        JSON.generate(file_path: expanded_file_path, agent_name: @agent_name, prompt_name: @prompt_name)
+      )
+      ::File.join(agent_cache_dir, "#{key}.json")
+    end
+
+    def agent_cache_dir
+      cache_root = ENV['XDG_CACHE_HOME'] || ::File.expand_path('~/.cache')
+      ::File.join(cache_root, 'rcm', 'agents')
+    end
+
+    def expanded_file_path = ::File.expand_path(@file_path)
+
+    def current_time_ns
+      now = ::Time.now
+      (now.to_i * 1_000_000_000) + now.nsec
+    end
+
+    def skip_fresh_agent_processing!
+      info "Skipping #{@file_path} with agent #{@agent_name} and prompt #{@prompt_name}; agent cache is fresh"
+    end
+
+    def invalid_command_composition!(position)
+      raise InvalidComposition, "#{position} expects `from command ...`"
+    end
+
+    def command_failure_class = CommandFailed
+    def command_composition_subject = "file #{@file_path}"
   end
   # rubocop:enable Metrics/ClassLength
 
diff --git a/lib/dslkeywords/file_backup.rb b/lib/dslkeywords/file_backup.rb
index 210804c..1716aa5 100644
--- a/lib/dslkeywords/file_backup.rb
+++ b/lib/dslkeywords/file_backup.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest'
 
 module RCM
@@ -5,12 +7,16 @@ module RCM
   # Included by BasicFile so all file/directory/symlink resources share
   # the same backup logic.
   module FileBackup
+    def checksum_for(file_path)
+      Digest::SHA256.file(file_path).hexdigest
+    end
+
     # TODO: Make protected?
     def backup!(file_path, checksum = nil)
       return if @without_backup
 
       suffix = if ::File.file?(file_path)
-                 checksum.nil? ? Digest::SHA256.file(file_path).hexdigest : checksum
+                 checksum.nil? ? checksum_for(file_path) : checksum
                else
                  Time.now.strftime('%s-%L')
                end
@@ -18,8 +24,8 @@ module RCM
     end
 
     def different?(file_a, file_b)
-      checksum_a = Digest::SHA256.file(file_a).hexdigest
-      checksum_b = Digest::SHA256.file(file_b).hexdigest
+      checksum_a = checksum_for(file_a)
+      checksum_b = checksum_for(file_b)
       [checksum_a != checksum_b, checksum_a, checksum_b]
     end
 
diff --git a/lib/dslkeywords/prompt.rb b/lib/dslkeywords/prompt.rb
index 9e599ef..1b383b9 100644
--- a/lib/dslkeywords/prompt.rb
+++ b/lib/dslkeywords/prompt.rb
@@ -1,13 +1,20 @@
 # frozen_string_literal: true
 
+require_relative '../chained'
+require_relative 'command'
 require_relative 'keyword'
 
 module RCM
   # Stores a named prompt body for agent-backed file processing.
   class PromptDefinition < Keyword
+    include Chained
+    include CommandComposable
+
     attr_reader :name
 
     class InvalidName < StandardError; end
+    class InvalidComposition < StandardError; end
+    class CommandFailed < StandardError; end
 
     def self.id_for(name) = super(normalize_name(name))
 
@@ -20,6 +27,7 @@ module RCM
 
     def initialize(name)
       @name = self.class.normalize_name(name)
+      initialize_command_composition!
       super(@name)
     end
 
@@ -28,6 +36,37 @@ module RCM
 
       @text = value.to_s
     end
+
+    def from(value = nil) = value
+
+    def render(file_path) = render_composed_content(text.to_s, file_path)
+
+    def fingerprint_source
+      {
+        text: text.to_s,
+        composition: @command_composition.map { |entry| fingerprinted_command(entry) }
+      }
+    end
+
+    private
+
+    def fingerprinted_command(entry)
+      command_definition = dsl.object!(
+        CommandDefinition,
+        entry[:name],
+        error_class: DSL::NoSuchCommandDefinition,
+        kind: 'command'
+      )
+
+      entry.merge(template: command_definition.template.to_s)
+    end
+
+    def invalid_command_composition!(position)
+      raise InvalidComposition, "#{position} expects `from command ...`"
+    end
+
+    def command_failure_class = CommandFailed
+    def command_composition_subject = "prompt #{@name}"
   end
 
   # Adds the `prompt` definition keyword to the top-level DSL.