Module: LLM::Eval::Run

Included in:
LLM::Eval
Defined in:
lib/llm/eval/run.rb

Overview

Run-level report generation.

Instance Method Summary collapse

Instance Method Details

#run_aggregate

This method returns an undefined value.

Writes the run-level report, accumulating every sample.



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/llm/eval/run.rb', line 10

def run_aggregate
  dir = File.join(Dir.getwd, "runs", name.to_s, llm.name.to_s, model.to_s)
  FileUtils.mkdir_p(dir)
  json = File.join(dir, "run.json")
  data = if File.exist?(json)
    JSON.parse(File.read(json))
  else
    {
      "eval" => name,
      "provider" => llm.name.to_s,
      "model" => model.to_s,
      "samples" => []
    }
  end
  data["samples"] ||= []
  data["samples"].reject! { it["sample"] == sample }
  data["samples"] << run_sample_entry
  data["samples"].sort_by! { it["sample"] }
  data["score"] = data["samples"].sum { it["score"] } / data["samples"].size.to_f
  File.write(json, JSON.pretty_generate(data))
  File.write(File.join(dir, "run.txt"), run_text_report(data))
end