Documentation
¶
Index ¶
- func CalculateEvaluationStats(result *types.EvaluationResult)
- func CalculateRunSummary(r *types.EvaluationRun)
- func CalculateScore(expected types.ExpectedResults, actual []types.Issue) float64
- func ComparePrompts(resultsDir string) error
- func CompareResults(resultsDir string) error
- func LoadConfigs(path string) ([]types.EvaluationConfig, error)
- func LoadSuite(path string) (*types.EvaluationSuite, error)
- func PrintEvaluationSummary(r *types.EvaluationResult)
- func PrintMultiRunProgress(runNum, totalRuns int)
- func PrintRunHeader(modelName, promptVariant string, numRuns int)
- func PrintSummary(r *types.EvaluationRun)
- func PrintTestResult(result *types.TestCaseResult, err error)
- func SaveEvaluationResults(resultsDir string, result *types.EvaluationResult) error
- func ValidateConfig(config types.EvaluationConfig) error
- func ValidateConfigs(configs []types.EvaluationConfig) error
- func WarmUpModel(modelConfig llm.ProviderConfig) error
- type ComparisonResult
- type EvaluationConfig
- type EvaluationRun
- type Evaluator
- type ModelServerConfig
- type ServerConfig
- type ServerManager
- type TestCaseResult
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CalculateEvaluationStats ¶
func CalculateEvaluationStats(result *types.EvaluationResult)
func CalculateRunSummary ¶
func CalculateRunSummary(r *types.EvaluationRun)
func CalculateScore ¶
func CalculateScore(expected types.ExpectedResults, actual []types.Issue) float64
func ComparePrompts ¶
func CompareResults ¶
func LoadConfigs ¶
func LoadConfigs(path string) ([]types.EvaluationConfig, error)
func PrintEvaluationSummary ¶
func PrintEvaluationSummary(r *types.EvaluationResult)
func PrintMultiRunProgress ¶
func PrintMultiRunProgress(runNum, totalRuns int)
func PrintRunHeader ¶
func PrintSummary ¶
func PrintSummary(r *types.EvaluationRun)
func PrintTestResult ¶
func PrintTestResult(result *types.TestCaseResult, err error)
func SaveEvaluationResults ¶
func SaveEvaluationResults(resultsDir string, result *types.EvaluationResult) error
func ValidateConfig ¶
func ValidateConfig(config types.EvaluationConfig) error
func ValidateConfigs ¶
func ValidateConfigs(configs []types.EvaluationConfig) error
func WarmUpModel ¶
func WarmUpModel(modelConfig llm.ProviderConfig) error
WarmUpModel pre-loads the model by making a simple request to avoid initial loading time in evaluation
Types ¶
type ComparisonResult ¶
type EvaluationConfig ¶
type EvaluationConfig = types.EvaluationConfig
type EvaluationRun ¶
type EvaluationRun = types.EvaluationRun
type Evaluator ¶
type Evaluator struct {
// contains filtered or unexported fields
}
func (*Evaluator) RunEvaluation ¶
func (e *Evaluator) RunEvaluation(modelConfig llm.ProviderConfig, serverName string, promptVariant string, numRuns int) (*types.EvaluationResult, error)
func (*Evaluator) SaveEvaluationResults ¶
func (e *Evaluator) SaveEvaluationResults(result *types.EvaluationResult) error
type ModelServerConfig ¶
type ServerConfig ¶
type ServerConfig = types.ServerConfig
type ServerManager ¶
type ServerManager struct {
// contains filtered or unexported fields
}
func NewServerManager ¶
func NewServerManager(llamaServerPath string, port int, extraArgs ...string) *ServerManager
func (*ServerManager) IsRunning ¶
func (sm *ServerManager) IsRunning() bool
func (*ServerManager) StartServer ¶
func (sm *ServerManager) StartServer(modelPath string) error
func (*ServerManager) StopServer ¶
func (sm *ServerManager) StopServer()
type TestCaseResult ¶
type TestCaseResult = types.TestCaseResult
Source Files
Click to show internal directories.
Click to hide internal directories.