eval

type Bot func(in, out processor.Processor) *pipeline.Worker

type Event struct {
	// Kind is one of the Event* name constants.
	Kind string
	// Text is the reply text on an llm_response, the spoken text on a
	// tts_response, or the transcript on a user_transcription. Empty for events
	// that carry no text.
	Text string
	// Function is the tool name on a function_call.
	Function string
	// Args are the arguments the model produced for a function_call.
	Args map[string]any
}

type Expectation struct {
	// Event is the friendly event name to match (see the Event constants).
	Event string `yaml:"event"`
	// TextContains, when set, requires the event's text to contain this
	// substring, case-sensitively. Applies to llm_response and tts_response.
	TextContains string `yaml:"text_contains,omitempty"`
	// Eval, when set, is a natural-language criterion an LLM judge checks the
	// bot's reply against. Applies to llm_response and tts_response, the two
	// events carrying text the bot itself produced.
	Eval string `yaml:"eval,omitempty"`
	// Name is the single-call shorthand for Calls: the tool name a function_call
	// event must match.
	Name string `yaml:"name,omitempty"`
	// Args is the single-call shorthand for Calls: the argument subset the
	// matched call must carry.
	Args map[string]any `yaml:"args,omitempty"`
	// Calls is the set of tool calls the turn should make, for a function_call
	// event. They are matched by name in any order and the expectation passes
	// only when all of them are found. Built from `calls:`, or from the
	// `name:`/`args:` shorthand, by normalizeCalls.
	Calls []FunctionCall `yaml:"calls,omitempty"`
	// Absent inverts the expectation: it passes only when no event of this type
	// arrives before the WithinMS budget expires, and fails as soon as one does.
	// It matches on event type only, so no content check may accompany it.
	Absent bool `yaml:"absent,omitempty"`
	// WithinMS is the latency budget for the event, measured from the user input;
	// zero uses the harness default.
	WithinMS int `yaml:"within_ms,omitempty"`
	// contains filtered or unexported fields
}

type Failure struct {
	// Turn is the 1-based turn number.
	Turn int
	// Expectation is the 1-based expectation index within the turn.
	Expectation int
	// Event is the event name that was expected.
	Event string
	// Reason explains what went wrong.
	Reason string
}

type FunctionCall struct {
	// Name is the tool name to match. An empty name matches any call, which is
	// what a bare function_call expectation asserts.
	Name string `yaml:"name,omitempty"`
	// Args, when set, is a subset check on the call's arguments: every listed
	// key must be present with the listed value, and any further argument the
	// model passed is ignored.
	Args map[string]any `yaml:"args,omitempty"`
}

type Judge interface {
	// AddUserMessage records a user turn in the conversation.
	AddUserMessage(text string)
	// AddAssistantMessage appends a segment of the bot's current reply.
	AddAssistantMessage(text string)
	// Evaluate judges the conversation so far against criterion. A judge that
	// cannot answer reports VerdictNo with the reason, rather than failing: an
	// unavailable judge is a failed assertion, not a broken run.
	Evaluate(ctx context.Context, criterion string) JudgeVerdict
}

type JudgeVerdict struct {
	// Verdict is VerdictYes, VerdictNo or VerdictContinue.
	Verdict string
	// Reason is a one-sentence justification.
	Reason string
	// RawResponse is the judge model's raw text, for diagnostics.
	RawResponse string
}

type LLMJudge struct {
	// contains filtered or unexported fields
}

type Manifest struct {
	// Concurrency is how many scenarios run at once; zero uses a default.
	Concurrency int `yaml:"concurrency"`
	// Suite is the list of bots and their scenarios.
	Suite []SuiteEntry `yaml:"suite"`
	// contains filtered or unexported fields
}

type Options struct {
	// Judge grades `judge:` assertions; nil is fine when no scenario uses one.
	// It holds the conversation it grades against, so give each scenario its own
	// rather than reusing one across scenarios.
	Judge Judge
	// UserTTS selects audio mode. When set, each user turn is synthesized with
	// this TTS service and streamed to the bot as microphone audio (so the bot's
	// real VAD, turn detection and STT run), instead of the text-mode send-text.
	// Any jargo TTS service works, e.g. cartesia.NewTTS(...).
	UserTTS *tts.Base
	// OnProgress, when set, is called as each turn and expectation resolves, for
	// a caller reporting a long run as it happens rather than at the end. It runs
	// on the harness's own goroutine, so keep it quick.
	OnProgress func(Progress)
}

type Progress struct {
	// Turn is the 1-based turn number.
	Turn int
	// Expectation is the 1-based expectation index, or 0 for a turn-level
	// record: the turn's header, or a schedule that never fired.
	Expectation int
	// Event is the expectation's event name, or the turn's input for a header.
	Event string
	// Status is "turn", "matched", "failed" or "timeout".
	Status string
	// Detail is the failure reason, or what was matched.
	Detail string
}

type Result struct {
	// Scenario is the scenario's name.
	Scenario string
	// Failures lists every unmet expectation; empty means the scenario passed.
	Failures []Failure
	// Duration is how long the run took.
	Duration time.Duration
	// Events is every event the bot emitted, in order, whether or not a scenario
	// asserted on it. It is what a failure is read against: what the bot actually
	// did, rather than only what it was expected to do.
	Events []Event
	// DebugLog is a timestamped trace of the harness's own decisions: events as
	// they arrived, what each expectation waited for, what the judge said. It is
	// what makes a run that failed once and passed the next time diagnosable.
	DebugLog []string
}

type Scenario struct {
	// Name identifies the scenario in reports; required.
	Name string `yaml:"name"`
	// Turns are played in order.
	Turns []Turn `yaml:"turns"`
	// Context is the conversation the bot's context should start from. When set,
	// the harness sends it once the bot is ready, replacing whatever context the
	// bot built for itself. Leave it out to test the bot's own opening state.
	Context []frames.Message `yaml:"context,omitempty"`
	// contains filtered or unexported fields
}

type SendAfter struct {
	// Event is the event to schedule from, or empty for a pure delay.
	Event string `yaml:"event,omitempty"`
	// DelayMS is how long to wait after the event was seen, or, with no event,
	// after the previous turn's send.
	DelayMS int `yaml:"delay_ms,omitempty"`
}

type SuiteEntry struct {
	// BotURL is the bot's RTVI WebSocket endpoint (ws:// or wss://).
	BotURL string `yaml:"bot_url"`
	// Scenarios are scenario file paths, resolved relative to the manifest.
	Scenarios []string `yaml:"scenarios"`
}

type SuiteResult struct {
	// BotURL is the bot the scenario ran against.
	BotURL string
	// Scenario is the scenario file path.
	Scenario string
	// Result is the scenario result; zero-valued when Err is set.
	Result Result
	// Err is non-nil when the scenario could not be loaded or run.
	Err error
}

type Turn struct {
	// User is the text the user "says" this turn (sent as RTVI send-text).
	User string `yaml:"user,omitempty"`
	// DTMF is a keypad sequence the user presses, one frame per key. Mutually
	// exclusive with User. Quote it in YAML: an unquoted # starts a comment.
	DTMF string `yaml:"dtmf,omitempty"`
	// Expect lists the events to match, in order, after the turn's input.
	// Optional: a turn may just send input, with the assertion on a later turn.
	Expect []Expectation `yaml:"expect,omitempty"`
	// SendAfter, when set, schedules when the turn's input is sent rather than
	// sending it as soon as the previous turn finishes.
	SendAfter *SendAfter `yaml:"send_after,omitempty"`
	// contains filtered or unexported fields
}