Documentation
¶
Overview ¶
Package report provides report generation functionality for evaluation results.
Overview ¶
The report package processes NamespacedObserver trees containing ResultCollector data to produce markdown-formatted evaluation reports.
ByEval generates a report organized by evaluation type, then by model, then by test case. It requires observer paths following the /{model}/{test case}/{eval} structure.
Usage ¶
import "chainguard.dev/driftlessaf/agents/evals/report"
// Create some evaluation data
obs := evals.NewNamespacedObserver(func(name string) *evals.ResultCollector {
return evals.NewResultCollector(customObserver(name))
})
// Generate a report organized by evaluation type
reportStr, hasFailures := report.ByEval(obs, 0.8)
if hasFailures {
fmt.Printf("Report:\n%s", reportStr)
}
Report Format ¶
Reports are generated in markdown format with:
- A summary table of pass rates and grades per evaluation and model
- Hierarchical detail trees per evaluation
- Failure message lists
- Below-threshold grade details
Thread Safety ¶
ByEval is safe for concurrent use as it is a pure function that does not modify its input parameters. Multiple goroutines can safely call it simultaneously with the same or different observers.
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ByEval ¶
func ByEval(obs *evals.NamespacedObserver[*evals.ResultCollector], threshold float64) (string, bool)
ByEval generates a report organized by evaluation, then by model, then by test case. Assumes paths follow the pattern /{model}/{test case}/{eval} for paths with results. Returns the report string and a boolean indicating if any evaluations fell below the threshold.
Example ¶
ExampleByEval demonstrates basic usage of the ByEval report generator.
package main
import (
"fmt"
"chainguard.dev/driftlessaf/agents/evals"
"chainguard.dev/driftlessaf/agents/evals/report"
)
// exampleObserver implements evals.Observer for examples
type exampleObserver struct {
name string
count int64
}
func (e *exampleObserver) Fail(msg string) {
}
func (e *exampleObserver) Log(msg string) {
}
func (e *exampleObserver) Grade(score float64, reasoning string) {
}
func (e *exampleObserver) Increment() {
e.count++
}
func (e *exampleObserver) Total() int64 {
return e.count
}
func main() {
// Create a factory for result collectors
factory := func(name string) *evals.ResultCollector {
return evals.NewResultCollector(&exampleObserver{name: name})
}
// Create root observer
obs := evals.NewNamespacedObserver(factory)
// Add evaluation data following /{model}/{test case}/{eval} pattern
// Model: claude, Test case: security-check, Eval: no-vulnerabilities
evalObs1 := obs.Child("claude").Child("security-check").Child("no-vulnerabilities")
evalObs1.Fail("Buffer overflow detected")
evalObs1.Increment()
evalObs1.Increment()
// Model: gemini, Test case: security-check, Eval: no-vulnerabilities
evalObs2 := obs.Child("gemini").Child("security-check").Child("no-vulnerabilities")
evalObs2.Increment()
// Generate ByEval report with 80% threshold
reportStr, hasFailures := report.ByEval(obs, 0.8)
fmt.Printf("Has failures: %t\n", hasFailures)
fmt.Printf("Report:\n%s", reportStr)
}
Output: Has failures: true Report: ## Summary Table | Evaluation Metric | claude | gemini | Average | |----------------------|---------------|-------------|----------| | no-vulnerabilities | ❌ 50.0% | 100.0% | ❌ 75.0% | | └─ security-check | ❌ 0.50 (50%) | 1.00 (100%) | ❌ 75.0% | no-vulnerabilities [❌ 66.7%] (2/3) └ claude [❌ 50.0%] (1/2) └ security-check [❌ 50.0%] (1/2) └ 1 [FAIL] Buffer overflow detected
Example (MultipleEvaluations) ¶
ExampleByEval_multipleEvaluations demonstrates multiple evaluations organized by eval type.
package main
import (
"fmt"
"chainguard.dev/driftlessaf/agents/evals"
"chainguard.dev/driftlessaf/agents/evals/report"
)
// exampleObserver implements evals.Observer for examples
type exampleObserver struct {
name string
count int64
}
func (e *exampleObserver) Fail(msg string) {
}
func (e *exampleObserver) Log(msg string) {
}
func (e *exampleObserver) Grade(score float64, reasoning string) {
}
func (e *exampleObserver) Increment() {
e.count++
}
func (e *exampleObserver) Total() int64 {
return e.count
}
func main() {
// Create a factory for result collectors
factory := func(name string) *evals.ResultCollector {
return evals.NewResultCollector(&exampleObserver{name: name})
}
// Create root observer
obs := evals.NewNamespacedObserver(factory)
// Add data for multiple evaluations
// Security evaluation - passing
securityObs := obs.Child("claude").Child("auth-test").Child("security-check")
securityObs.Increment()
// Performance evaluation - passing
perfObs := obs.Child("claude").Child("load-test").Child("performance-check")
perfObs.Grade(0.85, "Good performance")
perfObs.Increment()
// Performance evaluation - failing (below 80% threshold)
perfFailObs := obs.Child("claude").Child("stress-test").Child("performance-check")
perfFailObs.Grade(0.65, "Performance issues under load")
perfFailObs.Increment()
// Generate report
reportStr, hasFailures := report.ByEval(obs, 0.8)
fmt.Printf("Has failures: %t\n", hasFailures)
fmt.Printf("Report:\n%s", reportStr)
}
Output: Has failures: true Report: ## Summary Table | Evaluation Metric | claude | Average | |-------------------|----------------|----------| | performance-check | ❌ 2/2 (75.0%) | ❌ 75.0% | | └─ stress-test | ❌ 0.65 (65%) | ❌ 65.0% | | security-check | 100.0% | 100.0% | performance-check [❌ 0.75 avg] (2 results) └ claude [❌ 0.75 avg] (2 results) └ stress-test [❌ 0.65 avg] (1 result) └ 1 [0.65] Performance issues under load security-check [100.0%] (1/1)
Example (WithGrades) ¶
ExampleByEval_withGrades demonstrates ByEval with graded evaluations.
package main
import (
"fmt"
"chainguard.dev/driftlessaf/agents/evals"
"chainguard.dev/driftlessaf/agents/evals/report"
)
// exampleObserver implements evals.Observer for examples
type exampleObserver struct {
name string
count int64
}
func (e *exampleObserver) Fail(msg string) {
}
func (e *exampleObserver) Log(msg string) {
}
func (e *exampleObserver) Grade(score float64, reasoning string) {
}
func (e *exampleObserver) Increment() {
e.count++
}
func (e *exampleObserver) Total() int64 {
return e.count
}
func main() {
// Create a factory for result collectors
factory := func(name string) *evals.ResultCollector {
return evals.NewResultCollector(&exampleObserver{name: name})
}
// Create root observer
obs := evals.NewNamespacedObserver(factory)
// Add evaluation data with grades
// Model: claude, Test case: code-quality, Eval: readability-score
evalObs1 := obs.Child("claude").Child("code-quality").Child("readability-score")
evalObs1.Grade(0.75, "Some improvements needed")
evalObs1.Increment()
// Model: gemini, Test case: code-quality, Eval: readability-score
evalObs2 := obs.Child("gemini").Child("code-quality").Child("readability-score")
evalObs2.Grade(0.90, "Very readable code")
evalObs2.Increment()
// Generate report with 80% threshold
reportStr, hasFailures := report.ByEval(obs, 0.8)
fmt.Printf("Has failures: %t\n", hasFailures)
fmt.Printf("Report:\n%s", reportStr)
}
Output: Has failures: true Report: ## Summary Table | Evaluation Metric | claude | gemini | Average | |--------------------|---------------|------------|---------| | readability-score | ❌ 75.0% | 90.0% | 82.5% | | └─ code-quality | ❌ 0.75 (75%) | 0.90 (90%) | 82.5% | readability-score [0.82 avg] (2 results) └ claude [❌ 0.75 avg] (1 result) └ code-quality [❌ 0.75 avg] (1 result) └ 1 [0.75] Some improvements needed
Types ¶
This section is empty.
Source Files
Click to show internal directories.
Click to hide internal directories.