setup

package
v0.2.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 29, 2026 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DefaultAgentName          = "eval-agent"
	DefaultToolCount          = 3
	DefaultMockTokens         = 500
	DefaultSystemPrompt       = "You are an evaluation agent. Use available tools when helpful, then answer concisely."
	DefaultRuntime            = RuntimeLocal
	DefaultMemoryUserID       = "eval-user"
	DefaultMemoryStoreMode    = memory.StoreModeOnDemand
	MemoryScenarioStoreRecall = "store_recall"
)

Variables

This section is empty.

Functions

func BuildAgent

func BuildAgent(cfg Config) (*agent.Agent, error)

BuildAgent constructs an agent from cfg using mock LLM and tools when not overridden.

func DefaultConfigPath

func DefaultConfigPath() string

DefaultConfigPath returns the default eval-harness config file path.

func MemoryAgentOption

func MemoryAgentOption(cfg Config) (agent.Option, error)

MemoryAgentOption returns WithMemory when memory is enabled.

func ParseMemoryStoreMode

func ParseMemoryStoreMode(raw string) (memory.StoreMode, error)

ParseMemoryStoreMode parses eval harness store mode strings.

func RegisterMockTools

func RegisterMockTools(count int, cfg ToolConfig, rng *rand.Rand) agent.ToolRegistry

RegisterMockTools registers count mock tools on a new registry.

Types

type Config

type Config struct {
	UserPrompt   string
	Runtime      Runtime
	Temporal     TemporalConfig
	AgentName    string
	SystemPrompt string
	LLM          LLMConfig
	Tool         ToolConfig
	ToolCount    int
	Memory       MemoryConfig
	LLMClient    interfaces.LLMClient
	ToolRegistry agent.ToolRegistry
	Logger       logger.Logger
}

Config holds settings for a single eval agent run.

func (*Config) ApplyDefaults

func (c *Config) ApplyDefaults()

ApplyDefaults fills unset config fields.

func (*Config) MemoryEnabled

func (c *Config) MemoryEnabled() bool

MemoryEnabled reports whether memory is wired for this run.

func (*Config) UseTemporal

func (c *Config) UseTemporal() bool

UseTemporal reports whether cfg selects the Temporal runtime.

func (*Config) UsesMemoryScenario

func (c *Config) UsesMemoryScenario() bool

UsesMemoryScenario reports whether the runner executes a multi-step memory scenario.

func (*Config) Validate

func (c *Config) Validate() error

Validate checks required config fields.

func (*Config) ValidateMemory

func (c *Config) ValidateMemory() error

ValidateMemory checks memory-related config when enabled.

type FileAgentConfig

type FileAgentConfig struct {
	Name         string `mapstructure:"name"`
	SystemPrompt string `mapstructure:"system_prompt"`
	ToolCount    int    `mapstructure:"tool_count"`
}

FileAgentConfig holds agent fields from YAML.

type FileConfig

type FileConfig struct {
	Runtime    string           `mapstructure:"runtime"`
	UserPrompt string           `mapstructure:"user_prompt"`
	Agent      FileAgentConfig  `mapstructure:"agent"`
	Memory     FileMemoryConfig `mapstructure:"memory"`
	Temporal   TemporalConfig   `mapstructure:"temporal"`
}

FileConfig is the YAML configuration for eval-harness runs.

func LoadConfig

func LoadConfig(path string) (*FileConfig, error)

LoadConfig reads and validates eval-harness config from a YAML file.

func (*FileConfig) Config

func (f *FileConfig) Config() Config

Config returns a runner Config from the file config.

type FileMemoryConfig

type FileMemoryConfig struct {
	Enabled      bool   `mapstructure:"enabled"`
	StoreMode    string `mapstructure:"store_mode"`
	UserID       string `mapstructure:"user_id"`
	Scenario     string `mapstructure:"scenario"`
	StorePrompt  string `mapstructure:"store_prompt"`
	RecallPrompt string `mapstructure:"recall_prompt"`
}

FileMemoryConfig holds memory fields from YAML.

type LLMConfig

type LLMConfig struct {
	MockTokens int
}

LLMConfig configures the built-in mock LLM (internal defaults, not in YAML).

type MemoryConfig

type MemoryConfig struct {
	Enabled      bool
	StoreMode    memory.StoreMode
	UserID       string
	Scenario     string
	StorePrompt  string
	RecallPrompt string
}

MemoryConfig configures long-term memory for eval harness runs.

func (*MemoryConfig) ApplyMemoryDefaults

func (m *MemoryConfig) ApplyMemoryDefaults()

ApplyMemoryDefaults fills unset memory config fields.

type MockLLMClient

type MockLLMClient struct {
	// contains filtered or unexported fields
}

MockLLMClient is a deterministic mock LLM for eval harness runs.

func NewMockLLMClient

func NewMockLLMClient(cfg LLMConfig, rng *rand.Rand) *MockLLMClient

NewMockLLMClient builds a mock LLM client from cfg.

func (*MockLLMClient) Generate

func (*MockLLMClient) GenerateStream

func (m *MockLLMClient) GenerateStream(ctx context.Context, request *interfaces.LLMRequest) (interfaces.LLMStream, error)

func (*MockLLMClient) GetModel

func (m *MockLLMClient) GetModel() string

func (*MockLLMClient) GetProvider

func (m *MockLLMClient) GetProvider() interfaces.LLMProvider

func (*MockLLMClient) IsStreamSupported

func (m *MockLLMClient) IsStreamSupported() bool

type MockTool

type MockTool struct {
	// contains filtered or unexported fields
}

MockTool is a mock tool for eval harness runs.

func (*MockTool) Description

func (t *MockTool) Description() string

func (*MockTool) DisplayName

func (t *MockTool) DisplayName() string

func (*MockTool) Execute

func (t *MockTool) Execute(ctx context.Context, args map[string]any) (any, error)

func (*MockTool) Name

func (t *MockTool) Name() string

func (*MockTool) Parameters

func (t *MockTool) Parameters() interfaces.JSONSchema

type Runtime

type Runtime string

Runtime selects the agent execution backend.

const (
	RuntimeLocal    Runtime = "local"
	RuntimeTemporal Runtime = "temporal"
)

type TemporalConfig

type TemporalConfig struct {
	Host      string `mapstructure:"host"`
	Port      int    `mapstructure:"port"`
	Namespace string `mapstructure:"namespace"`
	TaskQueue string `mapstructure:"task_queue"`
}

TemporalConfig configures Temporal when Runtime is temporal.

type ToolConfig

type ToolConfig struct{}

ToolConfig configures mock tools (internal defaults, not in YAML).

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL