eval_analyzer

package
v0.14.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 2, 2026 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func DetermineMergeStrategy

func DetermineMergeStrategy(issue IssueReport, similar []SimilarDoc, config DedupConfig) (MergeStrategy, *SimilarDoc)

DetermineMergeStrategy decides how to handle an issue given similar docs

func MergeDesignDoc

func MergeDesignDoc(existingPath string, issue IssueReport, totalFailures int) error

MergeDesignDoc merges new evidence into an existing design doc

Types

type AnalysisResult

type AnalysisResult struct {
	Issues       []IssueReport `json:"issues"`
	TotalRuns    int           `json:"total_runs"`
	FailureCount int           `json:"failure_count"`
	SuccessRate  float64       `json:"success_rate"`
}

AnalysisResult contains all issues discovered from eval results

type Analyzer

type Analyzer struct {
	// contains filtered or unexported fields
}

Analyzer aggregates eval results and identifies patterns

func NewAnalyzer

func NewAnalyzer(resultsDir string, minFrequency int, categories []string) *Analyzer

NewAnalyzer creates a new eval results analyzer

func (*Analyzer) Analyze

func (a *Analyzer) Analyze() (*AnalysisResult, error)

Analyze processes all eval results and returns discovered issues

type DedupConfig

type DedupConfig struct {
	Enabled            bool
	MergeThreshold     float64 // Similarity % for merging (0.0-1.0)
	ForceNew           bool    // Always create new docs
	SkipWellDocumented bool    // Skip if issue is already comprehensive
}

DedupConfig configures deduplication behavior

func DefaultDedupConfig

func DefaultDedupConfig() DedupConfig

DefaultDedupConfig returns default deduplication settings

type DesignDocData

type DesignDocData struct {
	// Header
	Title          string
	Date           string
	Frequency      int
	BenchmarkCount int
	Priority       string
	EstimatedLOC   string
	EstimatedTime  string
	Category       string
	Impact         string

	// Evidence
	Benchmarks        string
	Models            string
	TotalFailures     int
	FailurePercentage string
	ErrorExamples     []ErrorExample

	// Analysis
	ProblemStatement   string
	RootCause          string
	Solution           string
	ImplementationPlan string

	// Technical Design
	APIChanges        string
	TypeSystemChanges string
	RuntimeChanges    string

	// Implementation
	Tasks []Task

	// Testing
	UnitTests        string
	IntegrationTests string
	NewBenchmarks    string
	SuccessCriteria  []string

	// References
	SimilarFeatures   string
	RelatedDesignDocs string

	// Impact
	SuccessRateBefore     string
	TokenEfficiencyBefore string
	SuccessRateAfter      string
	TokenEfficiencyAfter  string

	// Metadata
	GeneratedDate  string
	GeneratorModel string
}

DesignDocData contains all data for rendering a design document

type DesignGenerator

type DesignGenerator struct {
	// contains filtered or unexported fields
}

DesignGenerator generates design documents from issue reports

func NewDesignGenerator

func NewDesignGenerator(model string, seed int64) (*DesignGenerator, error)

NewDesignGenerator creates a new design document generator

func (*DesignGenerator) Generate

func (g *DesignGenerator) Generate(ctx context.Context, issue IssueReport, totalFailures int) (string, error)

Generate creates a design document from an issue report

type ErrorExample

type ErrorExample struct {
	Index int
	Error string
	Code  string
	Lang  string
}

ErrorExample represents a single error case in the design doc

type ErrorExtractor

type ErrorExtractor struct {
	// contains filtered or unexported fields
}

ErrorExtractor parses stderr and code to identify specific issues

func NewErrorExtractor

func NewErrorExtractor() *ErrorExtractor

NewErrorExtractor creates a new error extractor with predefined patterns

func (*ErrorExtractor) EnhanceIssueReport

func (e *ErrorExtractor) EnhanceIssueReport(issue *IssueReport)

EnhanceIssueReport adds detailed error analysis to an issue report

func (*ErrorExtractor) Extract

func (e *ErrorExtractor) Extract(stderr, code string) *ParsedError

Extract analyzes stderr and returns parsed error information

type ErrorPattern

type ErrorPattern struct {
	Name        string
	Regex       *regexp.Regexp
	Category    string
	Severity    string
	Description string
}

ErrorPattern represents a recognized error pattern with extraction rules

type IssueReport

type IssueReport struct {
	Category      string   `json:"category"`       // "syntax_error", "type_error", "missing_feature", etc.
	Title         string   `json:"title"`          // Human-readable issue title
	Frequency     int      `json:"frequency"`      // How many evals hit this
	Benchmarks    []string `json:"benchmarks"`     // Which benchmarks failed
	Examples      []string `json:"examples"`       // Failed code examples
	ErrorMessages []string `json:"error_messages"` // Stderr from failures
	Impact        string   `json:"impact"`         // "critical", "high", "medium", "low"
	Lang          string   `json:"lang"`           // Language where issue occurred
	Models        []string `json:"models"`         // Models that encountered this
}

IssueReport represents a pattern of failures discovered in eval results

type MergeStrategy

type MergeStrategy string

MergeStrategy determines how to handle an issue when similar docs exist

const (
	StrategyCreate MergeStrategy = "create" // No similar doc exists
	StrategyMerge  MergeStrategy = "merge"  // Update existing doc
	StrategySkip   MergeStrategy = "skip"   // Already well-documented
	StrategyLink   MergeStrategy = "link"   // Related but distinct
)

type ParsedError

type ParsedError struct {
	Pattern    string            // Which pattern matched
	Category   string            // Error category
	Context    string            // Surrounding code context
	Suggestion string            // Suggested fix (if known)
	Metadata   map[string]string // Extracted fields (e.g., "missing_feature": "recursion")
}

ParsedError contains extracted information from an error message

type SimilarDoc

type SimilarDoc struct {
	Path            string
	Filename        string
	SimilarityScore float64
	Category        string
	Language        string
	Benchmarks      []string
	Frequency       int
}

SimilarDoc represents a design doc similar to the current issue

func FindSimilarDesignDocs

func FindSimilarDesignDocs(issue IssueReport, plannedDir string, config DedupConfig) ([]SimilarDoc, error)

FindSimilarDesignDocs searches for design docs similar to the given issue

type Task

type Task struct {
	Number      int
	Title       string
	LOC         string
	Time        string
	Description string
}

Task represents an implementation task

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL