Documentation
¶
Index ¶
- Constants
- func DetectProvider(model string) string
- func Has(name string) bool
- func List() []string
- func Register(name string, provider Provider)
- func RegisterFactory(name string, factory ProviderFactory)
- type AnthropicProvider
- func (p *AnthropicProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *AnthropicProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *AnthropicProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *AnthropicProvider) Name() string
- type CacheEntry
- type CompletionRequest
- type CompletionResponse
- type FunctionCall
- type GeminiProvider
- func (p *GeminiProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *GeminiProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *GeminiProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *GeminiProvider) Name() string
- type HuggingFaceProvider
- func (p *HuggingFaceProvider) ConnectMCPServer(ctx context.Context, config mcp.ServerConfig) error
- func (p *HuggingFaceProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *HuggingFaceProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *HuggingFaceProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *HuggingFaceProvider) Name() string
- type HuggingFaceStream
- type HuggingFaceStreamResponse
- type InferenceStream
- type InstrumentedConfig
- type InstrumentedProvider
- func (p *InstrumentedProvider) CreateCompletion(ctx context.Context, request CompletionRequest) (*CompletionResponse, error)
- func (p *InstrumentedProvider) CreateStreaming(ctx context.Context, request CompletionRequest) (Stream, error)
- func (p *InstrumentedProvider) CreateStructured(ctx context.Context, request StructuredRequest) (*StructuredResponse, error)
- func (p *InstrumentedProvider) Name() string
- type JSONSchemaValidator
- type Message
- type MockProvider
- func (m *MockProvider) AddCompletionResponse(response *CompletionResponse) *MockProvider
- func (m *MockProvider) AddError(err error) *MockProvider
- func (m *MockProvider) AddStreamChunks(chunks []*StreamChunk) *MockProvider
- func (m *MockProvider) AddStructuredResponse(response *StructuredResponse) *MockProvider
- func (m *MockProvider) CreateCompletion(ctx context.Context, request CompletionRequest) (*CompletionResponse, error)
- func (m *MockProvider) CreateStreaming(ctx context.Context, request CompletionRequest) (Stream, error)
- func (m *MockProvider) CreateStructured(ctx context.Context, request StructuredRequest) (*StructuredResponse, error)
- func (m *MockProvider) Name() string
- func (m *MockProvider) Reset()
- type MockStream
- type OpenAIProvider
- func (p *OpenAIProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *OpenAIProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *OpenAIProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *OpenAIProvider) Name() string
- type OptimizedHuggingFaceProvider
- func (p *OptimizedHuggingFaceProvider) ConnectMCPServer(ctx context.Context, config mcp.ServerConfig) error
- func (p *OptimizedHuggingFaceProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *OptimizedHuggingFaceProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *OptimizedHuggingFaceProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *OptimizedHuggingFaceProvider) EstimateTokens(text string) int
- func (p *OptimizedHuggingFaceProvider) GetMetrics() map[string]any
- func (p *OptimizedHuggingFaceProvider) Name() string
- type PerformanceMetrics
- type Provider
- type ProviderError
- type ProviderFactory
- type Registry
- type ResponseCache
- type SSEEvent
- type SSEStreamParser
- type Schema
- type SimulatedStream
- type Stream
- type StreamChunk
- type StreamCollector
- type StreamingInferenceService
- type StructuredOutputHandler
- type StructuredRequest
- type StructuredResponse
- type Tool
- type ToolCall
- type ToolCallDelta
- type Usage
- type ValidationResult
- type VertexAIProvider
- func (p *VertexAIProvider) Close() error
- func (p *VertexAIProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *VertexAIProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *VertexAIProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *VertexAIProvider) Name() string
- type XAIProvider
- func (p *XAIProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
- func (p *XAIProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
- func (p *XAIProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
- func (p *XAIProvider) Name() string
Constants ¶
const ( ErrorCodeInvalidRequest = "invalid_request" ErrorCodeAuthentication = "authentication_error" ErrorCodeRateLimit = "rate_limit_exceeded" ErrorCodeQuotaExceeded = "quota_exceeded" ErrorCodeServerError = "server_error" ErrorCodeTimeout = "timeout" ErrorCodeModelNotFound = "model_not_found" ErrorCodeContentFiltered = "content_filtered" ErrorCodeUnknown = "unknown_error" )
Common error codes
Variables ¶
This section is empty.
Functions ¶
func DetectProvider ¶
DetectProvider detects the provider type from model name
func List ¶
func List() []string
List returns all registered provider names from the global registry
func RegisterFactory ¶
func RegisterFactory(name string, factory ProviderFactory)
RegisterFactory registers a provider factory function
Types ¶
type AnthropicProvider ¶
type AnthropicProvider struct {
// contains filtered or unexported fields
}
AnthropicProvider implements Provider for Anthropic API
func NewAnthropicProvider ¶
func NewAnthropicProvider(apiKey, baseURL string) *AnthropicProvider
NewAnthropicProvider creates a new Anthropic provider
func (*AnthropicProvider) CreateCompletion ¶
func (p *AnthropicProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion
func (*AnthropicProvider) CreateStreaming ¶
func (p *AnthropicProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response
func (*AnthropicProvider) CreateStructured ¶
func (p *AnthropicProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response
func (*AnthropicProvider) Name ¶
func (p *AnthropicProvider) Name() string
Name returns the provider name
type CacheEntry ¶
type CacheEntry struct {
Response *CompletionResponse
Timestamp time.Time
}
CacheEntry represents a cached response
type CompletionRequest ¶
type CompletionRequest struct {
// Messages is the conversation history
Messages []Message `json:"messages"`
// Model is the model to use (e.g., "gpt-4", "claude-3-opus")
Model string `json:"model,omitempty"`
// Temperature controls randomness (0.0-2.0)
Temperature float64 `json:"temperature,omitempty"`
// MaxTokens is the maximum number of tokens to generate
MaxTokens int `json:"max_tokens,omitempty"`
// Tools available for the model to call
Tools []Tool `json:"tools,omitempty"`
// MaxIterations is the maximum number of ReAct iterations (for ReAct providers)
MaxIterations int `json:"max_iterations,omitempty"`
// TokenBudget is the total token budget for the entire conversation/loop
TokenBudget int `json:"token_budget,omitempty"`
// Additional provider-specific options
Extra map[string]any `json:"extra,omitempty"`
}
CompletionRequest represents a completion request
type CompletionResponse ¶
type CompletionResponse struct {
// Content is the generated text
Content string `json:"content"`
// FinishReason explains why generation stopped
FinishReason string `json:"finish_reason"`
// Usage contains token usage information
Usage Usage `json:"usage"`
// ToolCalls if the model called any tools
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
// Raw is the raw provider response for debugging
Raw any `json:"raw,omitempty"`
}
CompletionResponse represents a completion response
func MockCompletionResponse ¶
func MockCompletionResponse(content string) *CompletionResponse
MockCompletionResponse creates a mock completion response
type FunctionCall ¶
type FunctionCall struct {
Name string `json:"name"`
Arguments json.RawMessage `json:"arguments"`
}
FunctionCall represents a function call
type GeminiProvider ¶
type GeminiProvider struct {
// contains filtered or unexported fields
}
GeminiProvider implements Provider for Google Gemini API
func NewGeminiProvider ¶
func NewGeminiProvider(apiKey, baseURL string) *GeminiProvider
NewGeminiProvider creates a new Gemini provider
func (*GeminiProvider) CreateCompletion ¶
func (p *GeminiProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion
func (*GeminiProvider) CreateStreaming ¶
func (p *GeminiProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response
func (*GeminiProvider) CreateStructured ¶
func (p *GeminiProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response
type HuggingFaceProvider ¶
type HuggingFaceProvider struct {
// contains filtered or unexported fields
}
HuggingFaceProvider implements LLM provider for HuggingFace models with ReAct
func NewHuggingFaceProvider ¶
func NewHuggingFaceProvider(inf inference.InferenceService, model string) *HuggingFaceProvider
NewHuggingFaceProvider creates a new HuggingFace provider
func (*HuggingFaceProvider) ConnectMCPServer ¶
func (p *HuggingFaceProvider) ConnectMCPServer(ctx context.Context, config mcp.ServerConfig) error
ConnectMCPServer connects to an MCP server
func (*HuggingFaceProvider) CreateCompletion ¶
func (p *HuggingFaceProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion with ReAct tool calling
func (*HuggingFaceProvider) CreateStreaming ¶
func (p *HuggingFaceProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response (simulated for non-streaming inference)
func (*HuggingFaceProvider) CreateStructured ¶
func (p *HuggingFaceProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response with schema validation
func (*HuggingFaceProvider) Name ¶
func (p *HuggingFaceProvider) Name() string
Name returns the provider name
type HuggingFaceStream ¶
type HuggingFaceStream struct {
// contains filtered or unexported fields
}
HuggingFaceStream implements the Stream interface for HuggingFace providers
func NewHuggingFaceStream ¶
func NewHuggingFaceStream(ctx context.Context) *HuggingFaceStream
NewHuggingFaceStream creates a new streaming response handler
func (*HuggingFaceStream) Recv ¶
func (s *HuggingFaceStream) Recv() (*StreamChunk, error)
Recv receives the next chunk from the stream
func (*HuggingFaceStream) SendChunk ¶
func (s *HuggingFaceStream) SendChunk(chunk *StreamChunk) error
SendChunk sends a chunk to the stream
func (*HuggingFaceStream) SetError ¶
func (s *HuggingFaceStream) SetError(err error)
SetError sets an error on the stream
type HuggingFaceStreamResponse ¶
type HuggingFaceStreamResponse struct {
Token struct {
ID int `json:"id"`
Text string `json:"text"`
Logprob float64 `json:"logprob"`
Special bool `json:"special"`
} `json:"token"`
GeneratedText string `json:"generated_text,omitempty"`
Details *struct {
FinishReason string `json:"finish_reason"`
Tokens int `json:"generated_tokens"`
} `json:"details,omitempty"`
}
HuggingFaceStreamResponse represents a streaming response chunk from HuggingFace
type InferenceStream ¶
type InferenceStream interface {
Recv() (*inference.GenerateResponse, error)
Close() error
}
InferenceStream represents a streaming inference response
type InstrumentedConfig ¶
type InstrumentedConfig struct {
// Calculator for cost tracking (defaults to cost.DefaultCalculator)
Calculator *cost.Calculator
// Enabled controls whether instrumentation is active
Enabled bool
}
InstrumentedConfig contains configuration for instrumented providers
type InstrumentedProvider ¶
type InstrumentedProvider struct {
// contains filtered or unexported fields
}
InstrumentedProvider wraps a Provider with automatic observability and cost tracking. All LLM calls are automatically instrumented with: - Token usage tracking - Cost calculation - Performance metrics - Error tracking
func NewInstrumentedProvider ¶
func NewInstrumentedProvider(provider Provider, config *InstrumentedConfig) *InstrumentedProvider
NewInstrumentedProvider wraps a provider with automatic observability
func (*InstrumentedProvider) CreateCompletion ¶
func (p *InstrumentedProvider) CreateCompletion(ctx context.Context, request CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion with automatic instrumentation
func (*InstrumentedProvider) CreateStreaming ¶
func (p *InstrumentedProvider) CreateStreaming(ctx context.Context, request CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response with instrumentation
func (*InstrumentedProvider) CreateStructured ¶
func (p *InstrumentedProvider) CreateStructured(ctx context.Context, request StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response with automatic instrumentation
func (*InstrumentedProvider) Name ¶
func (p *InstrumentedProvider) Name() string
Name returns the underlying provider name
type JSONSchemaValidator ¶
type JSONSchemaValidator struct {
// contains filtered or unexported fields
}
JSONSchemaValidator validates JSON data against a JSON Schema
func NewJSONSchemaValidator ¶
func NewJSONSchemaValidator(strict bool) *JSONSchemaValidator
NewJSONSchemaValidator creates a new schema validator
func (*JSONSchemaValidator) Validate ¶
func (v *JSONSchemaValidator) Validate(schema *Schema, data any) *ValidationResult
Validate validates data against the schema
type Message ¶
type Message struct {
Role string `json:"role"` // "system", "user", "assistant"
Content string `json:"content"` // The message content
}
Message represents a chat message
type MockProvider ¶
type MockProvider struct {
// Responses to return for each request
CompletionResponses []*CompletionResponse
StructuredResponses []*StructuredResponse
StreamChunks [][]*StreamChunk
Errors []error
// Track calls
CompletionCalls []CompletionRequest
StructuredCalls []StructuredRequest
StreamCalls []CompletionRequest
// contains filtered or unexported fields
}
MockProvider is a mock LLM provider for testing
func NewMockProvider ¶
func NewMockProvider(name string) *MockProvider
NewMockProvider creates a new mock provider
func (*MockProvider) AddCompletionResponse ¶
func (m *MockProvider) AddCompletionResponse(response *CompletionResponse) *MockProvider
AddCompletionResponse adds a completion response to return
func (*MockProvider) AddError ¶
func (m *MockProvider) AddError(err error) *MockProvider
AddError adds an error to return
func (*MockProvider) AddStreamChunks ¶
func (m *MockProvider) AddStreamChunks(chunks []*StreamChunk) *MockProvider
AddStreamChunks adds stream chunks to return
func (*MockProvider) AddStructuredResponse ¶
func (m *MockProvider) AddStructuredResponse(response *StructuredResponse) *MockProvider
AddStructuredResponse adds a structured response to return
func (*MockProvider) CreateCompletion ¶
func (m *MockProvider) CreateCompletion(ctx context.Context, request CompletionRequest) (*CompletionResponse, error)
CreateCompletion implements Provider
func (*MockProvider) CreateStreaming ¶
func (m *MockProvider) CreateStreaming(ctx context.Context, request CompletionRequest) (Stream, error)
CreateStreaming implements Provider
func (*MockProvider) CreateStructured ¶
func (m *MockProvider) CreateStructured(ctx context.Context, request StructuredRequest) (*StructuredResponse, error)
CreateStructured implements Provider
type MockStream ¶
type MockStream struct {
// contains filtered or unexported fields
}
MockStream is a mock stream implementation
type OpenAIProvider ¶
type OpenAIProvider struct {
// contains filtered or unexported fields
}
OpenAIProvider implements Provider for OpenAI API
func NewOpenAIProvider ¶
func NewOpenAIProvider(apiKey, baseURL string) *OpenAIProvider
NewOpenAIProvider creates a new OpenAI provider
func (*OpenAIProvider) CreateCompletion ¶
func (p *OpenAIProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion
func (*OpenAIProvider) CreateStreaming ¶
func (p *OpenAIProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response
func (*OpenAIProvider) CreateStructured ¶
func (p *OpenAIProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response
type OptimizedHuggingFaceProvider ¶
type OptimizedHuggingFaceProvider struct {
// contains filtered or unexported fields
}
OptimizedHuggingFaceProvider implements an optimized LLM provider for HuggingFace models
func NewOptimizedHuggingFaceProvider ¶
func NewOptimizedHuggingFaceProvider(inf inference.InferenceService, model string) *OptimizedHuggingFaceProvider
NewOptimizedHuggingFaceProvider creates a new optimized provider
func (*OptimizedHuggingFaceProvider) ConnectMCPServer ¶
func (p *OptimizedHuggingFaceProvider) ConnectMCPServer(ctx context.Context, config mcp.ServerConfig) error
func (*OptimizedHuggingFaceProvider) CreateCompletion ¶
func (p *OptimizedHuggingFaceProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates an optimized completion with ReAct tool calling
func (*OptimizedHuggingFaceProvider) CreateStreaming ¶
func (p *OptimizedHuggingFaceProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
func (*OptimizedHuggingFaceProvider) CreateStructured ¶
func (p *OptimizedHuggingFaceProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
func (*OptimizedHuggingFaceProvider) EstimateTokens ¶
func (p *OptimizedHuggingFaceProvider) EstimateTokens(text string) int
EstimateTokens estimates token count (simple implementation)
func (*OptimizedHuggingFaceProvider) GetMetrics ¶
func (p *OptimizedHuggingFaceProvider) GetMetrics() map[string]any
GetMetrics returns current performance metrics
func (*OptimizedHuggingFaceProvider) Name ¶
func (p *OptimizedHuggingFaceProvider) Name() string
type PerformanceMetrics ¶
type PerformanceMetrics struct {
TotalRequests int64
SuccessfulCalls int64
FailedCalls int64
CacheHits int64
AverageLatency time.Duration
TokensProcessed int64
ToolCallAccuracy float64
// contains filtered or unexported fields
}
PerformanceMetrics tracks performance metrics
type Provider ¶
type Provider interface {
// CreateCompletion creates a completion (unstructured text response)
CreateCompletion(ctx context.Context, request CompletionRequest) (*CompletionResponse, error)
// CreateStructured creates a structured response with schema validation
CreateStructured(ctx context.Context, request StructuredRequest) (*StructuredResponse, error)
// CreateStreaming creates a streaming response
CreateStreaming(ctx context.Context, request CompletionRequest) (Stream, error)
// Name returns the provider name (e.g., "openai", "anthropic")
Name() string
}
Provider defines the interface for LLM providers
func CreateProvider ¶
CreateProvider creates a provider from a factory
func UnwrapProvider ¶
UnwrapProvider returns the underlying provider if wrapped, otherwise returns the provider as-is
func WrapProvider ¶
WrapProvider wraps a provider with instrumentation if not already wrapped
type ProviderError ¶
type ProviderError struct {
Provider string `json:"provider"`
Code string `json:"code"`
Message string `json:"message"`
Type string `json:"type,omitempty"`
StatusCode int `json:"status_code,omitempty"`
IsRetryable bool `json:"is_retryable"`
OriginalError error `json:"-"`
}
ProviderError represents a provider-specific error
func NewProviderError ¶
func NewProviderError(provider, code, message string, original error) *ProviderError
NewProviderError creates a new provider error
func (*ProviderError) Error ¶
func (e *ProviderError) Error() string
Error implements the error interface
func (*ProviderError) Unwrap ¶
func (e *ProviderError) Unwrap() error
Unwrap returns the original error
type ProviderFactory ¶
ProviderFactory is a function that creates a new provider instance
type Registry ¶
type Registry struct {
// contains filtered or unexported fields
}
Registry manages LLM providers
type ResponseCache ¶
type ResponseCache struct {
// contains filtered or unexported fields
}
ResponseCache caches recent completions for faster responses
type SSEStreamParser ¶
type SSEStreamParser struct {
// contains filtered or unexported fields
}
SSEStreamParser parses Server-Sent Events (SSE) streams from HuggingFace API
func NewSSEStreamParser ¶
func NewSSEStreamParser(r io.Reader) *SSEStreamParser
NewSSEStreamParser creates a new SSE parser
func (*SSEStreamParser) ReadEvent ¶
func (p *SSEStreamParser) ReadEvent() (*SSEEvent, error)
ReadEvent reads the next SSE event
type Schema ¶
type Schema struct {
Type string `json:"type,omitempty"`
Properties map[string]*Schema `json:"properties,omitempty"`
Required []string `json:"required,omitempty"`
Items *Schema `json:"items,omitempty"`
Enum []any `json:"enum,omitempty"`
Minimum *float64 `json:"minimum,omitempty"`
Maximum *float64 `json:"maximum,omitempty"`
MinLength *int `json:"minLength,omitempty"`
MaxLength *int `json:"maxLength,omitempty"`
Pattern string `json:"pattern,omitempty"`
Description string `json:"description,omitempty"`
Default any `json:"default,omitempty"`
OneOf []*Schema `json:"oneOf,omitempty"`
AnyOf []*Schema `json:"anyOf,omitempty"`
AllOf []*Schema `json:"allOf,omitempty"`
}
Schema represents a JSON Schema
func ParseSchema ¶
func ParseSchema(raw json.RawMessage) (*Schema, error)
ParseSchema parses a JSON Schema from raw JSON
func SchemaFromStruct ¶
SchemaFromStruct generates a JSON Schema from a Go struct type
type SimulatedStream ¶
type SimulatedStream struct {
// contains filtered or unexported fields
}
SimulatedStream creates a simulated stream from a complete response This is useful for providers that don't support native streaming
func NewSimulatedStream ¶
func NewSimulatedStream(content string, finishReason string, chunkSize int) *SimulatedStream
NewSimulatedStream creates a stream that simulates streaming from a complete response
func (*SimulatedStream) Close ¶
func (s *SimulatedStream) Close() error
Close closes the simulated stream
func (*SimulatedStream) Recv ¶
func (s *SimulatedStream) Recv() (*StreamChunk, error)
Recv receives the next simulated chunk
type Stream ¶
type Stream interface {
// Recv receives the next chunk
Recv() (*StreamChunk, error)
// Close closes the stream
Close() error
}
Stream represents a streaming response
type StreamChunk ¶
type StreamChunk struct {
// Delta is the incremental content
Delta string `json:"delta"`
// FinishReason if this is the last chunk
FinishReason string `json:"finish_reason,omitempty"`
// ToolCallDeltas if tools are being called
ToolCallDeltas []ToolCallDelta `json:"tool_call_deltas,omitempty"`
}
StreamChunk represents a chunk in a streaming response
func ParseHuggingFaceStreamChunk ¶
func ParseHuggingFaceStreamChunk(data string) (*StreamChunk, error)
ParseHuggingFaceStreamChunk parses a HuggingFace streaming response chunk
type StreamCollector ¶
type StreamCollector struct {
// contains filtered or unexported fields
}
StreamCollector collects chunks from a stream into a complete response
func NewStreamCollector ¶
func NewStreamCollector() *StreamCollector
NewStreamCollector creates a new stream collector
func (*StreamCollector) Collect ¶
func (c *StreamCollector) Collect(stream Stream) (*CompletionResponse, error)
Collect reads all chunks from a stream and returns the complete response
func (*StreamCollector) GetChunks ¶
func (c *StreamCollector) GetChunks() []*StreamChunk
GetChunks returns all collected chunks
type StreamingInferenceService ¶
type StreamingInferenceService interface {
inference.InferenceService
GenerateStream(ctx context.Context, req inference.GenerateRequest) (InferenceStream, error)
}
StreamingInferenceService extends InferenceService with streaming support
type StructuredOutputHandler ¶
type StructuredOutputHandler struct {
// contains filtered or unexported fields
}
StructuredOutputHandler handles structured output generation with schema validation
func NewStructuredOutputHandler ¶
func NewStructuredOutputHandler(inf inference.InferenceService, strict bool) *StructuredOutputHandler
NewStructuredOutputHandler creates a new structured output handler
func (*StructuredOutputHandler) Generate ¶
func (h *StructuredOutputHandler) Generate(ctx context.Context, req StructuredRequest, model string) (*StructuredResponse, error)
Generate generates a structured output with schema validation
type StructuredRequest ¶
type StructuredRequest struct {
CompletionRequest
// ResponseSchema is the JSON Schema for the expected response
ResponseSchema json.RawMessage `json:"response_schema"`
// ResponseFormat specifies the format (e.g., "json_object", "json_schema")
ResponseFormat string `json:"response_format,omitempty"`
// StrictSchema enables strict schema adherence (provider-dependent)
StrictSchema bool `json:"strict_schema,omitempty"`
}
StructuredRequest represents a request for structured output
type StructuredResponse ¶
type StructuredResponse struct {
// Data is the parsed structured data
Data json.RawMessage `json:"data"`
// Raw completion response
CompletionResponse
}
StructuredResponse represents a structured response
func MockStructuredResponse ¶
func MockStructuredResponse(data any) *StructuredResponse
MockStructuredResponse creates a mock structured response
type Tool ¶
type Tool struct {
Name string `json:"name"`
Description string `json:"description"`
Parameters json.RawMessage `json:"parameters"` // JSON Schema for parameters
}
Tool represents a function/tool that can be called by the LLM
type ToolCall ¶
type ToolCall struct {
ID string `json:"id"`
Type string `json:"type"` // "function"
Function FunctionCall `json:"function"`
Arguments json.RawMessage `json:"arguments,omitempty"`
}
ToolCall represents a function call made by the model
type ToolCallDelta ¶
type ToolCallDelta struct {
Index int `json:"index"`
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
FunctionName string `json:"function_name,omitempty"`
ArgumentDelta string `json:"argument_delta,omitempty"`
}
ToolCallDelta represents an incremental tool call update
type Usage ¶
type Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
Usage represents token usage information
type ValidationResult ¶
type ValidationResult struct {
Valid bool `json:"valid"`
Errors []string `json:"errors,omitempty"`
}
ValidationResult contains the result of schema validation
type VertexAIProvider ¶
type VertexAIProvider struct {
// contains filtered or unexported fields
}
VertexAIProvider implements Provider for Google Vertex AI using the Gen AI SDK
func NewVertexAIProvider ¶
func NewVertexAIProvider(projectID, location string) (*VertexAIProvider, error)
NewVertexAIProvider creates a new Vertex AI provider using the Google Gen AI SDK. It uses Application Default Credentials (ADC) for authentication.
Security: All API calls respect the context deadline. Callers should set appropriate timeouts (recommended: 60-120s for completion, 180s for streaming).
func (*VertexAIProvider) Close ¶
func (p *VertexAIProvider) Close() error
Close implements the Provider interface. The genai.Client does not provide a Close() method as of v0.5.0, so this is a no-op. The underlying HTTP client and resources are managed by the SDK and will be released when the Client is garbage collected.
func (*VertexAIProvider) CreateCompletion ¶
func (p *VertexAIProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion using the Gen AI SDK
func (*VertexAIProvider) CreateStreaming ¶
func (p *VertexAIProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response
func (*VertexAIProvider) CreateStructured ¶
func (p *VertexAIProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response with JSON schema
func (*VertexAIProvider) Name ¶
func (p *VertexAIProvider) Name() string
Name returns the provider name
type XAIProvider ¶
type XAIProvider struct {
// contains filtered or unexported fields
}
XAIProvider implements Provider for X.AI (Grok) API
func NewXAIProvider ¶
func NewXAIProvider(apiKey, model, baseURL string) *XAIProvider
NewXAIProvider creates a new X.AI provider
func (*XAIProvider) CreateCompletion ¶
func (p *XAIProvider) CreateCompletion(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
CreateCompletion creates a completion
func (*XAIProvider) CreateStreaming ¶
func (p *XAIProvider) CreateStreaming(ctx context.Context, req CompletionRequest) (Stream, error)
CreateStreaming creates a streaming response
func (*XAIProvider) CreateStructured ¶
func (p *XAIProvider) CreateStructured(ctx context.Context, req StructuredRequest) (*StructuredResponse, error)
CreateStructured creates a structured response