Documentation
¶
Overview ¶
Package record 提供高级测试录制和回放功能
Package record 提供测试的录制和回放功能 ¶
本文件实现图执行的录制和回放:
- GraphCassette: 图执行交互录制会话
- GraphInteraction: 单次图节点执行记录
- GraphRecorder: 图执行录制器
- GraphReplayer: 图执行回放器
Package record 提供测试的录制和回放功能 ¶
Record 系统允许录制和回放 LLM 调用,便于测试和调试:
- Recorder: 录制 LLM 调用
- Replayer: 回放录制的调用
- Cassette: 存储录制的会话
Index ¶
- type AssertionHelper
- func (h *AssertionHelper) AssertContains(haystack, needle string, message string) bool
- func (h *AssertionHelper) AssertEqual(expected, actual any, message string) bool
- func (h *AssertionHelper) AssertNotEmpty(value any, message string) bool
- func (h *AssertionHelper) Errors() []string
- func (h *AssertionHelper) HasErrors() bool
- type Cassette
- type Fixture
- type FixtureManager
- type GraphCassette
- func (c *GraphCassette) AddInteraction(interaction *GraphInteraction)
- func (c *GraphCassette) FindByName(nodeName string) []*GraphInteraction
- func (c *GraphCassette) FindByNode(nodeID string) []*GraphInteraction
- func (c *GraphCassette) Get(index int) (*GraphInteraction, error)
- func (c *GraphCassette) Len() int
- func (c *GraphCassette) Save(path string) error
- type GraphInteraction
- type GraphRecorder
- type GraphReplayer
- type Interaction
- type RAGCassette
- type RAGInteraction
- type Recorder
- func (r *Recorder) Cassette() *Cassette
- func (r *Recorder) Complete(ctx context.Context, req llm.CompletionRequest) (*llm.CompletionResponse, error)
- func (r *Recorder) CountTokens(messages []llm.Message) (int, error)
- func (r *Recorder) CountTokensContext(ctx context.Context, messages []llm.Message) (int, error)
- func (r *Recorder) Models() []llm.ModelInfo
- func (r *Recorder) Name() string
- func (r *Recorder) Save(path string) error
- func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) (*llm.Stream, error)
- type ReplayMode
- type Replayer
- func (r *Replayer) Complete(ctx context.Context, req llm.CompletionRequest) (*llm.CompletionResponse, error)
- func (r *Replayer) CountTokens(messages []llm.Message) (int, error)
- func (r *Replayer) CountTokensContext(ctx context.Context, messages []llm.Message) (int, error)
- func (r *Replayer) Models() []llm.ModelInfo
- func (r *Replayer) Name() string
- func (r *Replayer) Stats() (hits, misses int)
- func (r *Replayer) Stream(ctx context.Context, req llm.CompletionRequest) (*llm.Stream, error)
- type ReplayerOption
- type ScenarioManager
- type SessionEvent
- type SessionRecorder
- func (r *SessionRecorder) RecordEvent(eventType string, data map[string]any)
- func (r *SessionRecorder) RecordLLM(interaction Interaction)
- func (r *SessionRecorder) RecordRAG(interaction RAGInteraction)
- func (r *SessionRecorder) RecordTool(interaction ToolInteraction)
- func (r *SessionRecorder) SaveAll(basePath string) error
- type TestGenerator
- type TestScenario
- type TestStep
- type ToolCassette
- type ToolInteraction
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type AssertionHelper ¶
type AssertionHelper struct {
// contains filtered or unexported fields
}
AssertionHelper 断言辅助工具
func (*AssertionHelper) AssertContains ¶
func (h *AssertionHelper) AssertContains(haystack, needle string, message string) bool
AssertContains 断言包含
func (*AssertionHelper) AssertEqual ¶
func (h *AssertionHelper) AssertEqual(expected, actual any, message string) bool
AssertEqual 断言相等
func (*AssertionHelper) AssertNotEmpty ¶
func (h *AssertionHelper) AssertNotEmpty(value any, message string) bool
AssertNotEmpty 断言非空
type Cassette ¶
type Cassette struct {
// Name 会话名称
Name string `json:"name"`
// Description 描述
Description string `json:"description,omitempty"`
// Interactions 交互列表
Interactions []Interaction `json:"interactions"`
// Metadata 元数据
Metadata map[string]any `json:"metadata,omitempty"`
// CreatedAt 创建时间
CreatedAt time.Time `json:"created_at"`
// UpdatedAt 更新时间
UpdatedAt time.Time `json:"updated_at"`
}
Cassette 表示一个录制会话
func (*Cassette) AddInteraction ¶
func (c *Cassette) AddInteraction(interaction Interaction)
AddInteraction 添加交互
func (*Cassette) FindByHash ¶
func (c *Cassette) FindByHash(hash string) *Interaction
FindByHash 通过请求哈希查找交互
type Fixture ¶
type Fixture struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Data map[string]any `json:"data"`
CreatedAt time.Time `json:"created_at"`
}
Fixture 测试固件
type FixtureManager ¶
type FixtureManager struct {
// contains filtered or unexported fields
}
FixtureManager 固件管理器
func NewFixtureManager ¶
func NewFixtureManager(basePath string) *FixtureManager
NewFixtureManager 创建固件管理器
type GraphCassette ¶
type GraphCassette struct {
// Name 录制会话名称
Name string `json:"name"`
// Interactions 交互记录列表
Interactions []*GraphInteraction `json:"interactions"`
// Metadata 元数据
Metadata map[string]any `json:"metadata,omitempty"`
// CreatedAt 创建时间
CreatedAt time.Time `json:"created_at"`
// UpdatedAt 最后更新时间
UpdatedAt time.Time `json:"updated_at"`
// contains filtered or unexported fields
}
GraphCassette 图执行录制会话
存储完整的图执行过程,包含所有节点的执行记录。 支持序列化到文件和从文件加载,便于回放和调试。
func LoadGraphCassette ¶
func LoadGraphCassette(path string) (*GraphCassette, error)
LoadGraphCassette 从文件加载图执行录制
func NewGraphCassette ¶
func NewGraphCassette(name string) *GraphCassette
NewGraphCassette 创建图执行录制会话
func (*GraphCassette) AddInteraction ¶
func (c *GraphCassette) AddInteraction(interaction *GraphInteraction)
AddInteraction 添加交互记录
func (*GraphCassette) FindByName ¶
func (c *GraphCassette) FindByName(nodeName string) []*GraphInteraction
FindByName 查找指定名称的所有交互记录
func (*GraphCassette) FindByNode ¶
func (c *GraphCassette) FindByNode(nodeID string) []*GraphInteraction
FindByNode 查找指定节点的所有交互记录
返回节点 ID 匹配的所有交互记录,按添加顺序排列。 同一节点可能被多次执行(例如循环图),因此可能返回多条记录。
func (*GraphCassette) Get ¶
func (c *GraphCassette) Get(index int) (*GraphInteraction, error)
Get 获取指定索引的交互记录
func (*GraphCassette) Save ¶
func (c *GraphCassette) Save(path string) error
Save 保存到文件
自动创建所需目录。使用 JSON 格式,带缩进以便阅读。
type GraphInteraction ¶
type GraphInteraction struct {
// ID 交互唯一标识
ID string `json:"id"`
// NodeID 节点 ID
NodeID string `json:"node_id"`
// NodeName 节点名称
NodeName string `json:"node_name"`
// Input 节点输入状态
Input map[string]any `json:"input,omitempty"`
// Output 节点输出状态
Output map[string]any `json:"output,omitempty"`
// NextNode 下一个执行的节点 ID
NextNode string `json:"next_node,omitempty"`
// Duration 节点执行耗时
Duration time.Duration `json:"duration"`
// Timestamp 执行时间戳
Timestamp time.Time `json:"timestamp"`
// Error 执行错误信息(如果有)
Error string `json:"error,omitempty"`
}
GraphInteraction 图执行交互记录
记录图中单个节点的执行信息,包括输入、输出、下一个节点等。
type GraphRecorder ¶
type GraphRecorder struct {
// contains filtered or unexported fields
}
GraphRecorder 图执行录制器
用于在图执行过程中录制每个节点的执行情况。 线程安全:所有方法都是并发安全的。
func NewGraphRecorder ¶
func NewGraphRecorder(cassette *GraphCassette) *GraphRecorder
NewGraphRecorder 创建图执行录制器
func (*GraphRecorder) Cassette ¶
func (r *GraphRecorder) Cassette() *GraphCassette
Cassette 返回关联的录制会话
func (*GraphRecorder) RecordNode ¶
func (r *GraphRecorder) RecordNode(nodeID, nodeName string, input, output map[string]any, next string, dur time.Duration, err error)
RecordNode 录制节点执行
参数:
- nodeID: 节点 ID
- nodeName: 节点名称
- input: 节点输入状态
- output: 节点输出状态
- next: 下一个执行的节点 ID(空字符串表示结束)
- dur: 执行耗时
- err: 执行错误(nil 表示成功)
type GraphReplayer ¶
type GraphReplayer struct {
// contains filtered or unexported fields
}
GraphReplayer 图执行回放器
从录制的 GraphCassette 中按节点 ID 回放执行结果。 支持同一节点多次执行的场景(使用访问计数器)。
func NewGraphReplayer ¶
func NewGraphReplayer(cassette *GraphCassette) *GraphReplayer
NewGraphReplayer 创建图执行回放器
func (*GraphReplayer) ReplayNode ¶
func (r *GraphReplayer) ReplayNode(nodeID string) (output map[string]any, nextNode string, err error)
ReplayNode 回放节点执行结果
返回录制的输出、下一个节点和错误。 对于同一节点的多次调用,会按录制顺序依次返回。
type Interaction ¶
type Interaction struct {
// ID 交互 ID
ID string `json:"id"`
// Request 请求
Request llm.CompletionRequest `json:"request"`
// Response 响应
Response *llm.CompletionResponse `json:"response,omitempty"`
// Error 错误信息
Error string `json:"error,omitempty"`
// Duration 耗时
Duration time.Duration `json:"duration"`
// Timestamp 时间戳
Timestamp time.Time `json:"timestamp"`
// RequestHash 请求哈希(用于匹配)
RequestHash string `json:"request_hash"`
}
Interaction 表示一次 LLM 交互
type RAGCassette ¶
type RAGCassette struct {
Name string `json:"name"`
Interactions []RAGInteraction `json:"interactions"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
RAGCassette RAG 录制会话
func LoadRAGCassette ¶
func LoadRAGCassette(path string) (*RAGCassette, error)
LoadRAGCassette 从文件加载
func (*RAGCassette) AddInteraction ¶
func (c *RAGCassette) AddInteraction(interaction RAGInteraction)
AddInteraction 添加交互
func (*RAGCassette) FindByHash ¶
func (c *RAGCassette) FindByHash(hash string) *RAGInteraction
FindByHash 查找交互
type RAGInteraction ¶
type RAGInteraction struct {
ID string `json:"id"`
Operation string `json:"operation"` // retrieve, rerank, synthesize
Query string `json:"query"`
Documents []rag.Document `json:"documents,omitempty"`
Result string `json:"result,omitempty"`
Error string `json:"error,omitempty"`
Duration time.Duration `json:"duration"`
Timestamp time.Time `json:"timestamp"`
RequestHash string `json:"request_hash"`
}
RAGInteraction RAG 交互记录
type Recorder ¶
type Recorder struct {
// contains filtered or unexported fields
}
Recorder 录制 LLM 调用
func NewRecorder ¶
NewRecorder 创建录制器
func (*Recorder) Complete ¶
func (r *Recorder) Complete(ctx context.Context, req llm.CompletionRequest) (*llm.CompletionResponse, error)
Complete 执行并录制请求
func (*Recorder) CountTokens ¶
CountTokens 计算 Token 数
func (*Recorder) CountTokensContext ¶ added in v0.5.13
CountTokensContext 使用底层 Provider 的可取消能力计算 Token 数量。 若底层仅支持旧接口,则快速返回不支持错误,不执行无法中途取消的同步降级。
type ReplayMode ¶
type ReplayMode string
ReplayMode 回放模式
const ( // ReplayModeStrict 严格模式:必须找到匹配的录制 ReplayModeStrict ReplayMode = "strict" // ReplayModeFallback 回退模式:找不到时使用真实 Provider ReplayModeFallback ReplayMode = "fallback" )
type Replayer ¶
type Replayer struct {
// contains filtered or unexported fields
}
Replayer 回放录制的 LLM 调用
func NewReplayer ¶
func NewReplayer(cassette *Cassette, opts ...ReplayerOption) *Replayer
NewReplayer 创建回放器
func (*Replayer) Complete ¶
func (r *Replayer) Complete(ctx context.Context, req llm.CompletionRequest) (*llm.CompletionResponse, error)
Complete 回放或执行请求
func (*Replayer) CountTokens ¶
CountTokens 计算 Token 数
func (*Replayer) CountTokensContext ¶ added in v0.5.13
CountTokensContext 使用回退 Provider 的可取消能力或本地估算计算 Token 数量。 回退 Provider 仅支持旧接口时快速返回不支持错误,避免伪装调用期间可取消。
type ReplayerOption ¶
type ReplayerOption func(*Replayer)
ReplayerOption Replayer 选项
func WithFallbackProvider ¶
func WithFallbackProvider(provider llm.Provider) ReplayerOption
WithFallbackProvider 设置回退 Provider
type ScenarioManager ¶
type ScenarioManager struct {
// contains filtered or unexported fields
}
ScenarioManager 场景管理器
func NewScenarioManager ¶
func NewScenarioManager(basePath string) *ScenarioManager
NewScenarioManager 创建场景管理器
func (*ScenarioManager) Load ¶
func (m *ScenarioManager) Load(name string) (*TestScenario, error)
Load 加载场景
func (*ScenarioManager) Save ¶
func (m *ScenarioManager) Save(scenario *TestScenario) error
Save 保存场景
type SessionEvent ¶
type SessionEvent struct {
Type string `json:"type"` // llm, tool, rag, custom
Timestamp time.Time `json:"timestamp"`
Data map[string]any `json:"data"`
}
SessionEvent 会话事件
type SessionRecorder ¶
type SessionRecorder struct {
// contains filtered or unexported fields
}
SessionRecorder 会话录制器
录制完整的测试会话,包括所有类型的交互
func NewSessionRecorder ¶
func NewSessionRecorder(name string) *SessionRecorder
NewSessionRecorder 创建会话录制器
func (*SessionRecorder) RecordEvent ¶
func (r *SessionRecorder) RecordEvent(eventType string, data map[string]any)
RecordEvent 录制自定义事件
func (*SessionRecorder) RecordLLM ¶
func (r *SessionRecorder) RecordLLM(interaction Interaction)
RecordLLM 录制 LLM 交互
func (*SessionRecorder) RecordRAG ¶
func (r *SessionRecorder) RecordRAG(interaction RAGInteraction)
RecordRAG 录制 RAG 交互
func (*SessionRecorder) RecordTool ¶
func (r *SessionRecorder) RecordTool(interaction ToolInteraction)
RecordTool 录制 Tool 交互
func (*SessionRecorder) SaveAll ¶
func (r *SessionRecorder) SaveAll(basePath string) error
SaveAll 保存所有录制
type TestGenerator ¶
type TestGenerator struct {
// contains filtered or unexported fields
}
TestGenerator 测试生成器
从录制生成测试代码
func NewTestGenerator ¶
func NewTestGenerator(packageName string) *TestGenerator
NewTestGenerator 创建测试生成器
func (*TestGenerator) GenerateFromCassette ¶
func (g *TestGenerator) GenerateFromCassette(cassette *Cassette) string
GenerateFromCassette 从录制生成测试代码
type TestScenario ¶
type TestScenario struct {
Name string `json:"name"`
Description string `json:"description"`
Steps []TestStep `json:"steps"`
Fixtures []string `json:"fixtures"`
Metadata map[string]any `json:"metadata,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
TestScenario 测试场景
type TestStep ¶
type TestStep struct {
Name string `json:"name"`
Action string `json:"action"` // call_llm, call_tool, assert, etc
Input map[string]any `json:"input,omitempty"`
Expected map[string]any `json:"expected,omitempty"`
Cassette string `json:"cassette,omitempty"`
Description string `json:"description,omitempty"`
}
TestStep 测试步骤
type ToolCassette ¶
type ToolCassette struct {
Name string `json:"name"`
Interactions []ToolInteraction `json:"interactions"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
ToolCassette Tool 录制会话
func LoadToolCassette ¶
func LoadToolCassette(path string) (*ToolCassette, error)
LoadToolCassette 从文件加载
func (*ToolCassette) AddInteraction ¶
func (c *ToolCassette) AddInteraction(interaction ToolInteraction)
AddInteraction 添加交互
func (*ToolCassette) FindByHash ¶
func (c *ToolCassette) FindByHash(hash string) *ToolInteraction
FindByHash 查找交互
type ToolInteraction ¶
type ToolInteraction struct {
ID string `json:"id"`
ToolName string `json:"tool_name"`
Args map[string]any `json:"args"`
Result string `json:"result"`
Error string `json:"error,omitempty"`
Duration time.Duration `json:"duration"`
Timestamp time.Time `json:"timestamp"`
RequestHash string `json:"request_hash"`
}
ToolInteraction Tool 交互记录