Documentation
¶
Index ¶
- Constants
- func CalculateModelCost(models map[string]*LLMModelConfig, modelName string, ...) (float64, error)
- func GetModelPricing(models map[string]*LLMModelConfig, modelName string) (promptPrice, completionPrice, cachePrice float64, err error)
- func ResolveModels(models map[string]*LLMModelConfig) error
- func SubstituteEnvValue(val interface{}) interface{}
- func SubstituteEnvVars(content string) string
- func WithReasoningEffort(e string) model.Option
- func WithToolStream(enable bool) model.Option
- type GLMChatModel
- func (g *GLMChatModel) Generate(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error)
- func (g *GLMChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error)
- func (g *GLMChatModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error)
- type LLMCallMeta
- type LLMCallOptions
- type LLMManager
- type LLMModel
- func (m *LLMModel) Generate(ctx context.Context, messages []*schema.Message) (*schema.Message, error)
- func (m *LLMModel) GetStats() (successCount, failureCount, totalWaitTime, totalDuration int64)
- func (m *LLMModel) IsDisabled() bool
- func (m *LLMModel) RecordStatus(success bool)
- func (m *LLMModel) ResetStats()
- type LLMModelConfig
- type ModelConfigProvider
- type ModelNamesProvider
- type ModelSelectionStrategy
- type ModelStats
Constants ¶
const ( // DefaultConcurrencyLimit is the default concurrency limit for each model. DefaultConcurrencyLimit = 10 // MaxModelFailures is the maximum consecutive failures before disabling a model. MaxModelFailures = 5 )
const (
KeyTimeoutRate = "timeoutRate"
)
Variables ¶
This section is empty.
Functions ¶
func CalculateModelCost ¶
func CalculateModelCost(models map[string]*LLMModelConfig, modelName string, inputTokens, outputTokens int) (float64, error)
CalculateModelCost calculates cost for a model based on token usage. Prices are per 1M tokens in config, so we divide by 1,000,000.
func GetModelPricing ¶
func GetModelPricing(models map[string]*LLMModelConfig, modelName string) (promptPrice, completionPrice, cachePrice float64, err error)
GetModelPricing returns pricing information for a model.
func ResolveModels ¶
func ResolveModels(models map[string]*LLMModelConfig) error
ResolveModels resolves the 'extend' field in LLM model configurations.
func SubstituteEnvValue ¶
func SubstituteEnvValue(val interface{}) interface{}
SubstituteEnvValue applies SubstituteEnvVars recursively to a value tree.
func SubstituteEnvVars ¶
SubstituteEnvVars replaces ${VAR_NAME} or ${VAR_NAME:default} with environment variable values.
func WithReasoningEffort ¶
func WithToolStream ¶
Types ¶
type GLMChatModel ¶
type GLMChatModel struct {
// contains filtered or unexported fields
}
func NewGLMChatModel ¶
func NewGLMChatModel(apiKey, baseURL, model string) *GLMChatModel
func (*GLMChatModel) WithTools ¶
func (g *GLMChatModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error)
type LLMCallMeta ¶
type LLMCallMeta struct {
ModelName string
ModelConfig *LLMModelConfig
SystemPrompt string
UserPrompt string
Messages []*schema.Message
Attempt int
Start time.Time
Duration time.Duration
}
LLMCallMeta captures per-attempt context for hooks.
type LLMCallOptions ¶
type LLMCallOptions struct {
UserID int
JobID int
Node string
OnAttempt func(meta *LLMCallMeta)
OnSuccess func(meta *LLMCallMeta, response *schema.Message)
OnFailure func(meta *LLMCallMeta, err error)
}
LLMCallOptions contains options for LLM calls.
type LLMManager ¶
type LLMManager struct {
// contains filtered or unexported fields
}
LLMManager manages multiple LLM models with dynamic model ordering.
func NewLLMManager ¶
func NewLLMManager(cfg ModelConfigProvider, getModels func() []string) (*LLMManager, error)
NewLLMManager creates a new LLM manager with a model provider.
func (*LLMManager) Call ¶
func (m *LLMManager) Call(ctx context.Context, systemPrompt, userPrompt string, opts *LLMCallOptions) (string, error)
Call invokes LLM with automatic failover based on getModels order.
func (*LLMManager) GetAllModelStats ¶
func (m *LLMManager) GetAllModelStats() []ModelStats
GetAllModelStats returns statistics for all models managed by this manager.
func (*LLMManager) GetModels ¶
func (m *LLMManager) GetModels() []*LLMModel
GetModels returns the list of models managed by this manager.
func (*LLMManager) ResetAllModelStats ¶
func (m *LLMManager) ResetAllModelStats()
ResetAllModelStats resets statistics for all models managed by this manager.
type LLMModel ¶
type LLMModel struct {
Config *LLMModelConfig
ChatModel model.ToolCallingChatModel
// contains filtered or unexported fields
}
LLMModel represents a single LLM model with concurrency control.
func GetLLMModel ¶
func GetLLMModel(cfg ModelConfigProvider, name string) (*LLMModel, error)
GetLLMModel retrieves or creates an LLM model by name with caching.
func (*LLMModel) Generate ¶
func (m *LLMModel) Generate(ctx context.Context, messages []*schema.Message) (*schema.Message, error)
Generate calls the LLM model with concurrency control and timeout.
func (*LLMModel) IsDisabled ¶
IsDisabled checks if a model is disabled.
func (*LLMModel) RecordStatus ¶
RecordStatus records a success/failure for a model and disables it if threshold is reached.
func (*LLMModel) ResetStats ¶
func (m *LLMModel) ResetStats()
ResetStats resets the statistics counters for this model.
type LLMModelConfig ¶
type LLMModelConfig struct {
Name string `yaml:"name,omitempty" mapstructure:"name"`
Extend interface{} `yaml:"extend,omitempty" mapstructure:"extend"` // string or []string
APIType string `yaml:"api_type,omitempty" mapstructure:"api_type"`
APIKey string `yaml:"api_key,omitempty" mapstructure:"api_key"`
BaseURL string `yaml:"base_url,omitempty" mapstructure:"base_url"`
Timeout int `yaml:"timeout,omitempty" mapstructure:"timeout"`
TimeoutRate float64 `yaml:"timeout_rate,omitempty" mapstructure:"timeout_rate"`
Temperature *float64 `yaml:"temperature,omitempty" mapstructure:"temperature"`
Concurrency int `yaml:"concurrency,omitempty" mapstructure:"concurrency"`
PromptPrice float64 `yaml:"prompt_price,omitempty" mapstructure:"prompt_price"`
CompletionPrice float64 `yaml:"completion_price,omitempty" mapstructure:"completion_price"`
CachePrice float64 `yaml:"cache_price,omitempty" mapstructure:"cache_price"`
Payload map[string]interface{} `yaml:"payload,omitempty" mapstructure:"payload"`
}
LLMModelConfig represents a single LLM model configuration.
func GetModelConfig ¶
func GetModelConfig(models map[string]*LLMModelConfig, name string) (*LLMModelConfig, error)
GetModelConfig returns LLM model configuration by name.
type ModelConfigProvider ¶
type ModelConfigProvider interface {
GetLLMModel(name string) (*LLMModelConfig, error)
}
ModelConfigProvider provides access to model configuration by name.
type ModelNamesProvider ¶
type ModelNamesProvider interface {
GetLLMModelNames() []string
}
ModelNamesProvider provides a default ordered model list.
type ModelSelectionStrategy ¶
type ModelSelectionStrategy string
ModelSelectionStrategy defines how models are selected.
const ( StrategySequential ModelSelectionStrategy = "sequential" StrategyRandom ModelSelectionStrategy = "random" )
Source Files