Documentation
¶
Overview ¶
Package llm provides a unified abstraction layer for Large Language Model interactions within the Mattermost AI plugin.
This package defines the core interfaces and data structures for working with various LLM providers (OpenAI, Anthropic, etc.) in a consistent manner. It handles:
- LanguageModel interface abstraction for different LLM providers
- Conversation management with structured posts, roles, and context
- Prompt template system with embedded templates and variable substitution
- Streaming text responses for real-time chat interactions
- Tool/function calling capabilities with JSON schema validation
- Request/response structures with token counting and truncation
- Context management including user info, channels, and bot configurations
The package is designed to be provider-agnostic, allowing the plugin to work with multiple LLM services through a common interface while preserving provider-specific capabilities like vision, JSON output, and tool calling.
Index ¶
- Constants
- func CreateTokenLogger() (*mlog.Logger, error)
- func GenerateBenchText(size int) string
- func IsValidService(service ServiceConfig) bool
- func NewJSONSchemaFromStruct[T any]() *jsonschema.Schema
- func SanitizeNonPrintableChars(s string) string
- type Annotation
- type AnnotationType
- type BenchmarkScenario
- type BotConfig
- type ChannelAccessLevel
- type CompletionRequest
- type Context
- type ContextOption
- type EventType
- type File
- type LanguageModel
- type LanguageModelConfig
- type LanguageModelLogWrapper
- func (w *LanguageModelLogWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
- func (w *LanguageModelLogWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
- func (w *LanguageModelLogWrapper) CountTokens(text string) int
- func (w *LanguageModelLogWrapper) InputTokenLimit() int
- type LanguageModelOption
- type LanguageModelTestLogWrapper
- func (w *LanguageModelTestLogWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
- func (w *LanguageModelTestLogWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
- func (w *LanguageModelTestLogWrapper) CountTokens(text string) int
- func (w *LanguageModelTestLogWrapper) InputTokenLimit() int
- type LanguageModelWrapper
- type MetricsObserver
- type ModelInfo
- type Post
- type PostRole
- type Prompts
- type ReasoningData
- type ServiceConfig
- type StreamGenerator
- type TextStreamEvent
- type TextStreamResult
- type TokenUsage
- type TokenUsageLoggingWrapper
- func (w *TokenUsageLoggingWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
- func (w *TokenUsageLoggingWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
- func (w *TokenUsageLoggingWrapper) CountTokens(text string) int
- func (w *TokenUsageLoggingWrapper) InputTokenLimit() int
- type Tool
- type ToolArgumentGetter
- type ToolAuthError
- type ToolCall
- type ToolCallStatus
- type ToolInfo
- type ToolResolver
- type ToolStore
- func (s *ToolStore) AddAuthError(authError ToolAuthError)
- func (s *ToolStore) AddTools(tools []Tool)
- func (s *ToolStore) GetAuthErrors() []ToolAuthError
- func (s *ToolStore) GetTools() []Tool
- func (s *ToolStore) GetToolsInfo() []ToolInfo
- func (s *ToolStore) ResolveTool(name string, argsGetter ToolArgumentGetter, context *Context) (string, error)
- func (s *ToolStore) TraceResolved(name string, argsGetter ToolArgumentGetter, result string, err error)
- func (s *ToolStore) TraceUnknown(name string, argsGetter ToolArgumentGetter)
- type TraceLog
- type TruncationWrapper
- func (w *TruncationWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
- func (w *TruncationWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
- func (w *TruncationWrapper) CountTokens(text string) int
- func (w *TruncationWrapper) InputTokenLimit() int
- type UserAccessLevel
Constants ¶
const ( ServiceTypeOpenAI = "openai" ServiceTypeOpenAICompatible = "openaicompatible" ServiceTypeAzure = "azure" ServiceTypeASage = "asage" ServiceTypeAnthropic = "anthropic" ServiceTypeCohere = "cohere" ServiceTypeBedrock = "bedrock" ServiceTypeMistral = "mistral" )
const FunctionsTokenBudget = 200
const MinTokens = 100
const PromptExtension = "tmpl"
const TokenLimitBufferSize = 0.9
Variables ¶
This section is empty.
Functions ¶
func CreateTokenLogger ¶ added in v1.4.0
CreateTokenLogger creates a dedicated logger for token usage metrics
func GenerateBenchText ¶ added in v1.7.0
GenerateBenchText creates a string of the specified size using a repeating pattern. Uses a realistic text pattern rather than random bytes.
func IsValidService ¶ added in v1.5.0
func IsValidService(service ServiceConfig) bool
IsValidService validates a service configuration
func NewJSONSchemaFromStruct ¶
func NewJSONSchemaFromStruct[T any]() *jsonschema.Schema
NewJSONSchemaFromStruct creates a JSONSchema from a Go struct using generics It's a helper function for tool providers that currently define schemas as structs
func SanitizeNonPrintableChars ¶ added in v1.7.0
SanitizeNonPrintableChars replaces non-printable Unicode characters with their escaped representation [U+XXXX] to prevent text spoofing attacks such as bidirectional text attacks that can make URLs appear to point to different domains. Uses [U+XXXX] format instead of \uXXXX to avoid JSON parsers converting it back. Allows newline, tab, and carriage return for JSON formatting. Also escapes variation selectors and other default ignorable code points which are technically "printable" but render invisibly and can be used for spoofing.
Types ¶
type Annotation ¶ added in v1.5.0
type Annotation struct {
Type AnnotationType `json:"type"` // Type of annotation
StartIndex int `json:"start_index"` // Start position in message text (0-based)
EndIndex int `json:"end_index"` // End position in message text (0-based)
URL string `json:"url"` // Source URL
Title string `json:"title"` // Source title
CitedText string `json:"cited_text,omitempty"` // Optional: text being cited (for context)
Index int `json:"index"` // Display index (1-based for UI)
}
Annotation represents an inline annotation/citation in the response text
type AnnotationType ¶ added in v1.5.0
type AnnotationType string
AnnotationType represents different types of annotations
const ( // AnnotationTypeURLCitation represents a web search citation AnnotationTypeURLCitation AnnotationType = "url_citation" )
type BenchmarkScenario ¶ added in v1.7.0
type BenchmarkScenario struct {
Name string
Generator StreamGenerator
}
BenchmarkScenario defines a benchmark test scenario
func BenchmarkScenarios ¶ added in v1.7.0
func BenchmarkScenarios() []BenchmarkScenario
BenchmarkScenarios returns common scenarios for stream benchmarks. This combines size-based scenarios with event-type scenarios.
type BotConfig ¶
type BotConfig struct {
ID string `json:"id"`
Name string `json:"name"`
DisplayName string `json:"displayName"`
CustomInstructions string `json:"customInstructions"`
ServiceID string `json:"serviceID"`
// Model is the optional model override for this bot.
// If not specified, the service's DefaultModel will be used.
Model string `json:"model"`
// Service is deprecated and kept only for backwards compatibility during migration.
Service *ServiceConfig `json:"service,omitempty"`
EnableVision bool `json:"enableVision"`
DisableTools bool `json:"disableTools"`
ChannelAccessLevel ChannelAccessLevel `json:"channelAccessLevel"`
ChannelIDs []string `json:"channelIDs"`
UserAccessLevel UserAccessLevel `json:"userAccessLevel"`
UserIDs []string `json:"userIDs"`
TeamIDs []string `json:"teamIDs"`
MaxFileSize int64 `json:"maxFileSize"`
// EnabledNativeTools contains the list of enabled native tools for this bot
// For OpenAI: ["web_search", "file_search", "code_interpreter"] (only works when UseResponsesAPI is true)
// For Anthropic: ["web_search"]
EnabledNativeTools []string `json:"enabledNativeTools"`
// ReasoningEnabled determines whether reasoning/thinking is enabled for this bot
// Applicable to OpenAI (with ResponsesAPI) and Anthropic
ReasoningEnabled bool `json:"reasoningEnabled"`
// ReasoningEffort determines the reasoning effort level for OpenAI models
// Valid values: "minimal", "low", "medium", "high"
// Only applicable to OpenAI with ResponsesAPI enabled
// Default: "medium"
ReasoningEffort string `json:"reasoningEffort"`
// ThinkingBudget determines the token budget for Anthropic thinking
// Must be at least 1024 and cannot exceed the OutputTokenLimit
// Only applicable to Anthropic
// Default: 1/4 of OutputTokenLimit, capped at 8192
ThinkingBudget int `json:"thinkingBudget"`
}
type ChannelAccessLevel ¶
type ChannelAccessLevel int
const ( ChannelAccessLevelAll ChannelAccessLevel = iota ChannelAccessLevelAllow ChannelAccessLevelBlock ChannelAccessLevelNone )
type CompletionRequest ¶
func (CompletionRequest) ExtractSystemMessage ¶
func (b CompletionRequest) ExtractSystemMessage() string
ExtractSystemMessage extracts the system message from the conversation.
func (CompletionRequest) String ¶
func (b CompletionRequest) String() string
type Context ¶
type Context struct {
// Server
Time string
ServerName string
CompanyName string
// Location
Team *model.Team
Channel *model.Channel
Thread []Post // Normalized posts that already have been formatted. nil if not in a thread or a root post
// User that is making the request
RequestingUser *model.User
// Bot Specific
BotName string
BotUsername string
BotUserID string
BotModel string
CustomInstructions string
Tools *ToolStore
DisabledToolsInfo []ToolInfo // Info about tools that are unavailable in the current context (e.g., DM-only tools in a channel)
Parameters map[string]interface{}
}
Context represents the data necessary to build the context of the LLM. For consumers none of the fields can be assumed to be present.
func NewContext ¶
func NewContext(opts ...ContextOption) *Context
NewContext creates a new Context with the given options
type ContextOption ¶
type ContextOption func(*Context)
ContextOption defines a function that configures a Context
type EventType ¶
type EventType int
EventType represents the type of event in the text stream
const ( // EventTypeText represents a text chunk event EventTypeText EventType = iota // EventTypeEnd represents the end of the stream EventTypeEnd // EventTypeError represents an error event EventTypeError // EventTypeToolCalls represents a tool call event EventTypeToolCalls // EventTypeReasoning represents a reasoning summary chunk event EventTypeReasoning // EventTypeReasoningEnd represents the end of reasoning summary EventTypeReasoningEnd // EventTypeAnnotations represents annotations/citations in the response EventTypeAnnotations // EventTypeUsage represents token usage data EventTypeUsage )
type LanguageModel ¶
type LanguageModel interface {
ChatCompletion(conversation CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
ChatCompletionNoStream(conversation CompletionRequest, opts ...LanguageModelOption) (string, error)
CountTokens(text string) int
InputTokenLimit() int
}
type LanguageModelConfig ¶
type LanguageModelLogWrapper ¶
type LanguageModelLogWrapper struct {
// contains filtered or unexported fields
}
func NewLanguageModelLogWrapper ¶
func NewLanguageModelLogWrapper(log pluginapi.LogService, wrapped LanguageModel) *LanguageModelLogWrapper
func (*LanguageModelLogWrapper) ChatCompletion ¶
func (w *LanguageModelLogWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
func (*LanguageModelLogWrapper) ChatCompletionNoStream ¶
func (w *LanguageModelLogWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
func (*LanguageModelLogWrapper) CountTokens ¶
func (w *LanguageModelLogWrapper) CountTokens(text string) int
func (*LanguageModelLogWrapper) InputTokenLimit ¶
func (w *LanguageModelLogWrapper) InputTokenLimit() int
type LanguageModelOption ¶
type LanguageModelOption func(*LanguageModelConfig)
func WithJSONOutput ¶
func WithJSONOutput[T any]() LanguageModelOption
func WithMaxGeneratedTokens ¶
func WithMaxGeneratedTokens(maxGeneratedTokens int) LanguageModelOption
func WithModel ¶
func WithModel(model string) LanguageModelOption
func WithReasoningDisabled ¶ added in v1.7.0
func WithReasoningDisabled() LanguageModelOption
func WithToolsDisabled ¶ added in v1.6.0
func WithToolsDisabled() LanguageModelOption
type LanguageModelTestLogWrapper ¶
type LanguageModelTestLogWrapper struct {
// contains filtered or unexported fields
}
func NewLanguageModelTestLogWrapper ¶
func NewLanguageModelTestLogWrapper(t *testing.T, wrapped LanguageModel) *LanguageModelTestLogWrapper
func (*LanguageModelTestLogWrapper) ChatCompletion ¶
func (w *LanguageModelTestLogWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
func (*LanguageModelTestLogWrapper) ChatCompletionNoStream ¶
func (w *LanguageModelTestLogWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
func (*LanguageModelTestLogWrapper) CountTokens ¶
func (w *LanguageModelTestLogWrapper) CountTokens(text string) int
func (*LanguageModelTestLogWrapper) InputTokenLimit ¶
func (w *LanguageModelTestLogWrapper) InputTokenLimit() int
type LanguageModelWrapper ¶
type LanguageModelWrapper func(LanguageModel) LanguageModel
type MetricsObserver ¶ added in v1.6.0
type MetricsObserver interface {
ObserveTokenUsage(botName, teamID, userID string, inputTokens, outputTokens int)
}
MetricsObserver defines the interface for observing token usage metrics
type Prompts ¶
type Prompts struct {
// contains filtered or unexported fields
}
type ReasoningData ¶ added in v1.5.0
type ReasoningData struct {
Text string // The reasoning/thinking text content
Signature string // Opaque verification signature from the model
}
ReasoningData represents the complete reasoning/thinking data including signature
type ServiceConfig ¶
type ServiceConfig struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
APIKey string `json:"apiKey"`
OrgID string `json:"orgId"`
DefaultModel string `json:"defaultModel"`
APIURL string `json:"apiURL"`
Region string `json:"region"` // For AWS Bedrock region
// AWS IAM credentials for Bedrock (optional, takes precedence over APIKey)
AWSAccessKeyID string `json:"awsAccessKeyID"`
AWSSecretAccessKey string `json:"awsSecretAccessKey"`
// Renaming the JSON field to inputTokenLimit would require a migration, leaving as is for now.
InputTokenLimit int `json:"tokenLimit"`
StreamingTimeoutSeconds int `json:"streamingTimeoutSeconds"`
SendUserID bool `json:"sendUserID"`
// Otherwise known as maxTokens
OutputTokenLimit int `json:"outputTokenLimit"`
// UseResponsesAPI determines whether to use the new OpenAI Responses API
// Only applicable to OpenAI and OpenAI-compatible services
UseResponsesAPI bool `json:"useResponsesAPI"`
}
type StreamGenerator ¶ added in v1.7.0
type StreamGenerator struct {
// TotalTextSize is the total bytes of text to generate
TotalTextSize int
// ChunkSize is the size of each text chunk
ChunkSize int
// IncludeReasoning adds reasoning events before text events
IncludeReasoning bool
// IncludeToolCalls ends with tool calls instead of normal end event
IncludeToolCalls bool
// IncludeUsage adds a usage event before end
IncludeUsage bool
// IncludeAnnotations adds annotation events
IncludeAnnotations bool
}
StreamGenerator creates synthetic streams for benchmarking. It generates TextStreamResult with configurable event patterns.
func (*StreamGenerator) Generate ¶ added in v1.7.0
func (g *StreamGenerator) Generate() *TextStreamResult
Generate creates a new TextStreamResult with synthetic events. The stream is generated in a goroutine and returned immediately.
type TextStreamEvent ¶
TextStreamEvent represents an event in the text stream
type TextStreamResult ¶
type TextStreamResult struct {
Stream <-chan TextStreamEvent
}
TextStreamResult represents a stream of text events
func NewStreamFromString ¶
func NewStreamFromString(text string) *TextStreamResult
func (*TextStreamResult) ReadAll ¶
func (t *TextStreamResult) ReadAll() (string, error)
type TokenUsage ¶ added in v1.4.0
type TokenUsage struct {
InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"`
}
TokenUsage represents token usage statistics for an LLM request
type TokenUsageLoggingWrapper ¶ added in v1.4.0
type TokenUsageLoggingWrapper struct {
// contains filtered or unexported fields
}
TokenUsageLoggingWrapper wraps a LanguageModel to log token usage
func NewTokenUsageLoggingWrapper ¶ added in v1.4.0
func NewTokenUsageLoggingWrapper(wrapped LanguageModel, botUsername string, tokenLogger *mlog.Logger, metrics MetricsObserver) *TokenUsageLoggingWrapper
NewTokenUsageLoggingWrapper creates a new wrapper that logs token usage
func (*TokenUsageLoggingWrapper) ChatCompletion ¶ added in v1.4.0
func (w *TokenUsageLoggingWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
ChatCompletion intercepts the streaming response to extract and log token usage
func (*TokenUsageLoggingWrapper) ChatCompletionNoStream ¶ added in v1.4.0
func (w *TokenUsageLoggingWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
ChatCompletionNoStream uses the streaming method internally, so token usage logging happens automatically when ReadAll() processes the intercepted stream
func (*TokenUsageLoggingWrapper) CountTokens ¶ added in v1.4.0
func (w *TokenUsageLoggingWrapper) CountTokens(text string) int
CountTokens delegates to the wrapped model
func (*TokenUsageLoggingWrapper) InputTokenLimit ¶ added in v1.4.0
func (w *TokenUsageLoggingWrapper) InputTokenLimit() int
InputTokenLimit delegates to the wrapped model
type Tool ¶
type Tool struct {
Name string
Description string
Schema any
Resolver ToolResolver
}
Tool represents a function that can be called by the language model during a conversation.
Each tool has a name, description, and schema that defines its parameters. These are passed to the LLM for it to understand what capabilities it has. It is the Resolver function that implements the actual functionality.
The Schema field should contain a JSONSchema that defines the expected structure of the tool's arguments. The Resolver function receives the conversation context and a way to access the parsed arguments, and returns either a result that will be passed to the LLM or an error.
type ToolArgumentGetter ¶
type ToolAuthError ¶ added in v1.4.0
type ToolAuthError struct {
ServerName string `json:"server_name"`
AuthURL string `json:"auth_url"`
Error error `json:"error"`
}
ToolAuthError represents an authentication error that occurred during tool creation
type ToolCall ¶
type ToolCall struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Arguments json.RawMessage `json:"arguments"`
Result string `json:"result"`
Status ToolCallStatus `json:"status"`
}
ToolCall represents a tool call. An empty result indicates that the tool has not yet been resolved.
func (*ToolCall) SanitizeArguments ¶ added in v1.7.0
func (tc *ToolCall) SanitizeArguments()
SanitizeArguments sanitizes the Arguments field to prevent bidirectional text and other Unicode spoofing attacks.
type ToolCallStatus ¶
type ToolCallStatus int
ToolCallStatus represents the current status of a tool call
const ( // ToolCallStatusPending indicates the tool is waiting for user approval/rejection ToolCallStatusPending ToolCallStatus = iota // ToolCallStatusAccepted indicates the user has accepted the tool call but it's not resolved yet ToolCallStatusAccepted // ToolCallStatusRejected indicates the user has rejected the tool call ToolCallStatusRejected // ToolCallStatusError indicates the tool call was accepted but errored during resolution ToolCallStatusError // ToolCallStatusSuccess indicates the tool call was accepted and resolved successfully ToolCallStatusSuccess )
type ToolInfo ¶ added in v1.7.0
ToolInfo represents basic information about a tool without its full implementation. Used to inform LLMs about tools that are unavailable in the current context.
type ToolResolver ¶
type ToolResolver func(context *Context, argsGetter ToolArgumentGetter) (string, error)
type ToolStore ¶
type ToolStore struct {
// contains filtered or unexported fields
}
func NewNoTools ¶
func NewNoTools() *ToolStore
func NewToolStore ¶
func (*ToolStore) AddAuthError ¶ added in v1.4.0
func (s *ToolStore) AddAuthError(authError ToolAuthError)
AddAuthError adds an authentication error to the tool store
func (*ToolStore) GetAuthErrors ¶ added in v1.4.0
func (s *ToolStore) GetAuthErrors() []ToolAuthError
GetAuthErrors returns all authentication errors collected during tool creation
func (*ToolStore) GetToolsInfo ¶ added in v1.7.0
GetToolsInfo returns basic information (name and description) about all tools in the store. This is useful for informing LLMs about tools that are available in other contexts (e.g., DM-only tools when in a channel).
func (*ToolStore) ResolveTool ¶
func (*ToolStore) TraceResolved ¶
func (s *ToolStore) TraceResolved(name string, argsGetter ToolArgumentGetter, result string, err error)
func (*ToolStore) TraceUnknown ¶
func (s *ToolStore) TraceUnknown(name string, argsGetter ToolArgumentGetter)
type TruncationWrapper ¶
type TruncationWrapper struct {
// contains filtered or unexported fields
}
func NewLLMTruncationWrapper ¶
func NewLLMTruncationWrapper(llm LanguageModel) *TruncationWrapper
func (*TruncationWrapper) ChatCompletion ¶
func (w *TruncationWrapper) ChatCompletion(request CompletionRequest, opts ...LanguageModelOption) (*TextStreamResult, error)
func (*TruncationWrapper) ChatCompletionNoStream ¶
func (w *TruncationWrapper) ChatCompletionNoStream(request CompletionRequest, opts ...LanguageModelOption) (string, error)
func (*TruncationWrapper) CountTokens ¶
func (w *TruncationWrapper) CountTokens(text string) int
func (*TruncationWrapper) InputTokenLimit ¶
func (w *TruncationWrapper) InputTokenLimit() int
type UserAccessLevel ¶
type UserAccessLevel int
const ( UserAccessLevelAll UserAccessLevel = iota UserAccessLevelAllow UserAccessLevelBlock UserAccessLevelNone )