Class: LLM::Eval

Inherits:
Object
  • Object
show all
Extended by:
FileUtils
Includes:
Run
Defined in:
lib/llm/eval.rb,
lib/llm/eval/run.rb,
lib/llm/eval/tracer.rb

Overview

The Eval class represents an eval for a given provider, and path to an eval.rb file that describes the eval.

Defined Under Namespace

Modules: Run, Tracer

Constant Summary collapse

File = 
::File
UNDEFINED =

This constant is part of a private API. You should avoid using this constant if possible, as it may be removed or be changed in the future. 

Object.new

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Run

#run_aggregate

Constructor Details

#initialize(&b) ⇒ LLM::Eval

Parameters:

  • b (Proc) 


65
66
67
68
69
70
71
 
# File 'lib/llm/eval.rb', line 65

def initialize(&b)
  @steps     = []
  @block     = b
  @result    = 1.0
  @final     = nil
  @penalties = []
end

Instance Attribute Details

#sample ⇒ Integer (readonly)

Returns:

  • (Integer) 


39
40
41
 
# File 'lib/llm/eval.rb', line 39

def sample
  @sample
end

#steps ⇒ Array<String> (readonly)

Returns:

  • (Array<String>) 


43
44
45
 
# File 'lib/llm/eval.rb', line 43

def steps
  @steps
end

#llm ⇒ LLM::Provider (readonly)

Returns:



47
48
49
 
# File 'lib/llm/eval.rb', line 47

def llm
  @llm
end

#model ⇒ String (readonly)

Returns:

  • (String) 


51
52
53
 
# File 'lib/llm/eval.rb', line 51

def model
  @model
end

#root ⇒ String (readonly)

Returns:

  • (String) 


55
56
57
 
# File 'lib/llm/eval.rb', line 55

def root
  @root
end

#penalties ⇒ Array<Hash> (readonly)

Returns The reasons the score came out below 1.0.

Returns:

  • (Array<Hash>) —

    The reasons the score came out below 1.0



60
61
62
 
# File 'lib/llm/eval.rb', line 60

def penalties
  @penalties
end

Class Method Details

.from(llm, model, path, sample = 1) ⇒ LLM::Eval

Parameters:

Returns:



27
28
29
30
31
32
33
34
35
 
# File 'lib/llm/eval.rb', line 27

def self.from(llm, model, path, sample=1)
  p = File.realpath(path)
  e = eval File.read(p), TOPLEVEL_BINDING, p, 1
  e.instance_variable_set(:@root, File.dirname(p))
  e.instance_variable_set(:@llm, llm)
  e.instance_variable_set(:@model, model)
  e.instance_variable_set(:@sample, sample)
  e
end

Instance Method Details

#penalize!(by:, why:)

This method returns an undefined value.

Reduce @result

Parameters:

  • by (Integer)
  • why (String) 


225
226
227
 
# File 'lib/llm/eval.rb', line 225

def penalize!(by:, why:)
  @penalties << {by:, why:}
end

#set(**options) ⇒ LLM::Eval

Parameters:

  • options (Hash)

Returns:



76
77
78
79
 
# File 'lib/llm/eval.rb', line 76

def set(**options)
  options.each { |key, value| public_send(key, value) }
  self
end

#configure

This method returns an undefined value.



83
84
85
 
# File 'lib/llm/eval.rb', line 83

def configure(&)
  instance_eval(&)
end

#name(name = UNDEFINED) ⇒ String?

Parameters:

  • name (String, nil) (defaults to: UNDEFINED)

Returns:

  • (String, nil) 


90
91
92
93
94
95
96
 
# File 'lib/llm/eval.rb', line 90

def name(name = UNDEFINED)
  if name == UNDEFINED
    @name
  else
    @name = name
  end
end

#description(desc = UNDEFINED) ⇒ String?

Parameters:

  • desc (String, nil) (defaults to: UNDEFINED)

Returns:

  • (String, nil) 


101
102
103
104
105
106
107
 
# File 'lib/llm/eval.rb', line 101

def description(desc = UNDEFINED)
  if desc == UNDEFINED
    @desc
  else
    @desc = desc
  end
end

#step(path) ⇒ Array<String>

Parameters:

  • path (String)

Returns:

  • (Array<String>) 


112
113
114
115
 
# File 'lib/llm/eval.rb', line 112

def step(path)
  target = File.join(@root, path)
  @steps << File.read(target)
end

#score(&block)

This method returns an undefined value.

Parameters:

  • block (Proc) —

    Receives the agent that ran and returns a Float between 0.0 and 1.0



122
123
124
125
126
127
128
 
# File 'lib/llm/eval.rb', line 122

def score(&block)
  if block
    @score = block
  else
    @score
  end
end

#result ⇒ Float?

Returns:

  • (Float, nil) 


132
133
134
 
# File 'lib/llm/eval.rb', line 132

def result
  @final ? @final.clamp(0.0, 1.0) : @final
end

#setup(&block) ⇒ Object

Parameters:

  • block (Proc) —

    A proc.



139
140
141
142
143
144
145
 
# File 'lib/llm/eval.rb', line 139

def setup(&block)
  if block
    @setup = block
  else
    @setup
  end
end

#teardown(&block) ⇒ Object

Parameters:

  • block (Proc) —

    A proc.



150
151
152
153
154
155
156
 
# File 'lib/llm/eval.rb', line 150

def teardown(&block)
  if block
    @teardown = block
  else
    @teardown
  end
end

#intercept(&block) ⇒ Object

Parameters:

  • block (Proc) —

    A proc.



161
162
163
164
165
166
167
 
# File 'lib/llm/eval.rb', line 161

def intercept(&block)
  if block
    @intercept = block
  else
    @intercept
  end
end

#tracer(&block) ⇒ Object

Parameters:

  • block (Proc) —

    A proc.



172
173
174
175
176
177
178
 
# File 'lib/llm/eval.rb', line 172

def tracer(&block)
  if block
    @tracer = block
  else
    @tracer
  end
end

#agent(&block)

This method returns an undefined value.

Configures the agent



183
184
185
186
187
188
189
 
# File 'lib/llm/eval.rb', line 183

def agent(&block)
  if block
    @agent = block
  else
    @agent
  end
end

#call ⇒ Array<Hash>

Returns:

  • (Array<Hash>) 


193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/llm/eval.rb', line 193

def call
  eval
  _agent = agent
  steps.each { _agent.talk(it, model:) }
  score.call(_agent)
  penalties.each { @result -= it[:by] }
  @final = @result
  if tracer
    tracer.on_exit
    run_aggregate
  end
  {name:, score: @result}
end

#eval

This method returns an undefined value.

Evals @block within self.



210
211
212
213
214
215
216
217
218
 
# File 'lib/llm/eval.rb', line 210

def eval
  instance_eval(&@block) if @block
  instance_eval(&@setup) if @setup
  @intercept = instance_eval(&@intercept) if @intercept
  @tracer = instance_eval(&@tracer) if @tracer
  @agent = @agent.call
ensure
  instance_eval(&@teardown) if @teardown
end