llmx

package module
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 10, 2026 License: MIT Imports: 28 Imported by: 0

README

llmx

A Go library for building with large language models.

⚠️ Work in progress.

Documentation

Index

Constants

View Source
const (
	ProviderProxy     = "proxy"
	ProviderOpenAI    = "openai"
	ProviderAnthropic = "anthropic"
	ProviderGoogle    = "google"
	ProviderBailian   = "bailian"
)
View Source
const OUTPUT_SCHEMA_INSTRUCTION = OutputSchemaInstruction
View Source
const OutputSchemaInstruction = "<|output_json_schema|>"
View Source
const ProviderOpenai = ProviderOpenAI

Variables

View Source
var (
	ErrMaxTurnExceeded = errors.New("max turn exceeded")
	ErrSessionClosed   = errors.New("session closed")
	ErrSessionStarted  = errors.New("session already started")
	ErrSessionBusy     = errors.New("session busy")
)
View Source
var DefaultModelInfos = map[string]ModelInfo{
	"gpt-5.4": {
		MaxToken:       1000000,
		MaxOutputToken: 128000,
	},
	"gpt-5.4-mini": {
		MaxToken:       400000,
		MaxOutputToken: 128000,
	},
	"gpt-5.4-nano": {
		MaxToken:       400000,
		MaxOutputToken: 128000,
	},
	"gemini-3.1-pro-preview": {
		MaxToken:       1064000,
		MaxOutputToken: 64000,
	},
	"gemini-3-flash-preview": {
		MaxToken:       1064000,
		MaxOutputToken: 64000,
	},
	"gemini-3-pro-image-preview": {
		MaxToken:       97000,
		MaxOutputToken: 32000,
	},
	"gemini-3.1-flash-image-preview": {
		MaxToken:       160000,
		MaxOutputToken: 32000,
	},
	"glm-5": {
		MaxToken:       200000,
		MaxOutputToken: 128000,
	},
	"glm-5.1": {
		MaxToken:       200000,
		MaxOutputToken: 128000,
	},
	"deepseek-chat": {
		MaxToken:       128000,
		MaxOutputToken: 8000,
	},
	"deepseek-reasoner": {
		MaxToken:       128000,
		MaxOutputToken: 64000,
	},
}

Functions

func ApplyCompression

func ApplyCompression(history []*types.Message, compress *EventCompression) ([]*types.Message, int64, error)

ApplyCompression applies one compression event to a history and returns the compressed history along with an estimate of how much context the compression removed. It is the single definition of what a compression means: Compress uses it to report the history a compression leaves behind, and a session uses it to restore that same history from the event log. The reduction is derived here instead of being stored in the event, so generating and replaying a compression always report the same value.

The given history is never modified.

func Completion

func Completion(ctx context.Context, model string, messages []*types.Message, options ...types.CompletionOption) (*types.Completion, error)

func DefaultModelsInitError

func DefaultModelsInitError() error

func Embedding

func Embedding(ctx context.Context, model string, input []string, options ...types.EmbeddingOption) (*types.Embedding, error)

func NewProvider

func NewProvider(name string, opts ...types.ProviderOption) (types.Provider, error)

func Realtime

func Realtime(ctx context.Context, model string, messages []*types.Message, options ...types.RealTimeOption) (types.RealTimeSession, error)

func RegistProviderCreator

func RegistProviderCreator(name string, Creator ProviderCreator)

func RegisterProviderCreator

func RegisterProviderCreator(name string, creator ProviderCreator) error

func Rerank

func Rerank(ctx context.Context, model string, input *types.RerankInput, options ...types.RerankOption) (*types.RerankOutput, error)

func RunCompletionTask

func RunCompletionTask[T any](ctx context.Context, m *Model, prompt string, failRetry int, options ...types.CompletionOption) (*T, error)

Types

type Agent

type Agent struct {
	Name        string
	Description string
	// contains filtered or unexported fields
}

func NewAgent

func NewAgent(
	name string,
	description string,
	models *Models,
	opts ...AgentOption,
) (*Agent, error)

func (*Agent) NewSession

func (a *Agent) NewSession(
	history []*Event,
	opts ...AgentOption,
) (*Session, error)

NewSession restores one reusable session without starting an execution.

func (*Agent) Run

func (a *Agent) Run(
	ctx context.Context,
	history []*Event,
	input *SessionInput,
	opts ...AgentOption,
) (*SessionResult, error)

Run creates a session and processes one fixed input synchronously.

func (*Agent) Start

func (a *Agent) Start(
	ctx context.Context,
	history []*Event,
	opts ...AgentOption,
) (*Runner, error)

Start creates a session in long-running mode. The returned runner initially waits for input and remains alive until ctx is canceled, LoopStop is returned, or an execution error occurs.

type AgentOption

type AgentOption func(*AgentOptions)

func WithAdditionalTools

func WithAdditionalTools(tools ...*Tool) AgentOption

WithAdditionalTools appends tools to those inherited from the Agent.

func WithCompress

func WithCompress(compress *CompressOptions) AgentOption

func WithEventSink

func WithEventSink(sink EventSink) AgentOption

func WithInstruction

func WithInstruction(instruction string) AgentOption

func WithLLMInterceptor

func WithLLMInterceptor(interceptors ...LLMInterceptor) AgentOption

func WithLoop

func WithLoop(loop Loop) AgentOption

func WithMaxTurn

func WithMaxTurn(maxTurn int) AgentOption

func WithModel

func WithModel(model string) AgentOption

func WithReasonLevel

func WithReasonLevel(level int) AgentOption

func WithStream

func WithStream(stream bool) AgentOption

func WithToolCallInterceptor

func WithToolCallInterceptor(interceptors ...ToolCallInterceptor) AgentOption

func WithTools

func WithTools(tools ...*Tool) AgentOption

WithTools replaces all tools inherited from the Agent.

type AgentOptions

type AgentOptions struct {
	Model                string
	Instruction          string
	Loop                 Loop
	ReasonLevel          int
	Tools                []*Tool
	MaxTurn              int
	Stream               bool
	Compress             *CompressOptions
	EventSink            EventSink
	LLMInterceptors      []LLMInterceptor
	ToolCallInterceptors []ToolCallInterceptor
}

func ApplyAgentOptions

func ApplyAgentOptions(options *AgentOptions, opts ...AgentOption) *AgentOptions

func DefaultAgentOptions

func DefaultAgentOptions() *AgentOptions

func (*AgentOptions) Clone

func (o *AgentOptions) Clone() *AgentOptions

type CompressInput

type CompressInput struct {
	Model    string
	Messages []*types.Message
	Options  *CompressOptions

	// Tokens is the context usage the model last reported. Compression is
	// decided exclusively from this value; zero means there is no reported
	// context size to compress yet.
	Tokens int64
}

type CompressOptions

type CompressOptions struct {
	// KeepLastTurn is the number of recent user turns left unchanged.
	KeepLastTurn int

	ToolResultLevel      float64
	ToolResultReplace    string
	ToolResultReplaceTag string

	SummaryLevel       float64
	SummaryModel       string
	SummaryInstruction string
	SummaryBridge      string
	SummaryBridgeTag   string
	SummaryTimeout     time.Duration
}

type CompressResultSummary

type CompressResultSummary struct {
	Message *types.Message        `json:"message,omitempty" yaml:"message,omitempty"`
	Usage   types.CompletionUsage `json:"usage,omitempty" yaml:"usage,omitempty"`
}

CompressResultSummary replaces the whole covered range with one bridge message. The messages it replaces stay in the event log, so only the bridge itself has to be stored here. Usage is from the summary call that wrote the bridge: it read the range the bridge replaces, so the reduction this compression achieves is derived from it when the event is applied.

type CompressResultToolResult

type CompressResultToolResult struct {
	Result       string            `json:"result,omitempty" yaml:"result,omitempty"`
	Instructions map[string]string `json:"instructions,omitempty" yaml:"instructions,omitempty"`
}

type Event

type Event struct {
	ID string `json:"id,omitempty" yaml:"id,omitempty"`

	Message       *types.Message      `json:"message,omitempty" yaml:"message,omitempty"`
	HumanRequest  *EventHumanRequest  `json:"humanRequest,omitempty" yaml:"humanRequest,omitempty"`
	HumanResponse *EventHumanResponse `json:"humanResponse,omitempty" yaml:"humanResponse,omitempty"`
	Compression   *EventCompression   `json:"compression,omitempty" yaml:"compression,omitempty"`
	Usage         *EventUsage         `json:"usage,omitempty" yaml:"usage,omitempty"`
	State         *EventState         `json:"state,omitempty" yaml:"state,omitempty"`
}

Event is a oneof envelope. Exactly one payload field must be non-nil. Once accepted by a Session, an Event and its nested values are immutable; callers must Clone before modifying them.

func Compress

func Compress(
	ctx context.Context,
	models *Models,
	in *CompressInput,
) ([]*Event, error)

Compress decides whether a history must shrink and describes the transforms that shrink it. It is stateless: it never reads or writes session storage and never changes the history it is given, it only returns the compression events describing what to do. Callers apply those events with ApplyCompression, so generating and replaying a compression always agree.

At most one compression runs per call: the highest threshold the reported context size exceeds wins, and a history that is still too large after it triggers the next compression on a later call. When both thresholds are equal the tool-result compression wins, being the cheaper of the two. It returns no events when nothing has to be compressed.

func NewEventCompression

func NewEventCompression(compression *EventCompression) *Event

func NewEventHumanRequest

func NewEventHumanRequest(request *EventHumanRequest) *Event

func NewEventHumanResponse

func NewEventHumanResponse(response *EventHumanResponse) *Event

func NewEventMessage

func NewEventMessage(message *types.Message) *Event

NewEventMessage wraps one message into one event: a message is the unit of idempotency and ordering, so it is never batched with others.

func NewEventState

func NewEventState(state map[string]string, patch bool) *Event

func NewEventUsage

func NewEventUsage(usage types.CompletionUsage) *Event

func (*Event) Clone

func (e *Event) Clone() *Event

func (*Event) Validate

func (e *Event) Validate() error

type EventCompression

type EventCompression struct {
	//one of
	ToolResult *CompressResultToolResult `json:"toolResult,omitempty" yaml:"toolResult,omitempty"`
	Summary    *CompressResultSummary    `json:"summary,omitempty" yaml:"summary,omitempty"`

	// UntilMessageID is the last history message this compression covers,
	// inclusive. Messages after it are left unchanged.
	UntilMessageID string `json:"untilMessageID,omitempty" yaml:"untilMessageID,omitempty"`
}

EventCompression is one self-contained compression: the range it covers and the transform it applies to that range. How much context the transform removed is not stored; ApplyCompression derives it from the event, so generating and replaying a compression always report the same reduction.

func (*EventCompression) Clone

func (e *EventCompression) Clone() *EventCompression

type EventHumanRequest

type EventHumanRequest struct {
	ID        string                 `json:"id,omitempty" yaml:"id,omitempty"`
	Arguments map[string]any         `json:"arguments,omitempty" yaml:"arguments,omitempty"`
	ToolCall  *types.MessageToolCall `json:"toolCall,omitempty" yaml:"toolCall,omitempty"`
}

func (*EventHumanRequest) Clone

type EventHumanResponse

type EventHumanResponse struct {
	ID     string         `json:"id,omitempty" yaml:"id,omitempty"`
	Result map[string]any `json:"result,omitempty" yaml:"result,omitempty"`
}

func (*EventHumanResponse) Clone

type EventSink

type EventSink func(context.Context, *Event) error

EventSink receives the immutable Event instance committed by a Session. Implementations must clone an Event before modifying it.

type EventState

type EventState struct {
	Patch bool              `json:"patch,omitempty" yaml:"patch,omitempty"`
	State map[string]string `json:"state,omitempty" yaml:"state,omitempty"`
}

type EventUsage

type EventUsage struct {
	Usage types.CompletionUsage `json:"usage,omitempty" yaml:"usage,omitempty"`
}

type LLMHandler

type LLMHandler func(context.Context, *SessionTurn) (*types.Completion, error)

type LLMInterceptor

type LLMInterceptor func(LLMHandler) LLMHandler

type Loop

type Loop func(ctx context.Context, turn *SessionTurnResult) (LoopAction, error)

Loop decides what a session does after every complete model turn, once all events produced by that turn have been committed.

type LoopAction

type LoopAction uint8

LoopAction tells a session what to do after one complete model turn.

const (
	// LoopWait yields control to the caller. Run returns, while Start waits for
	// the next input without closing the session.
	LoopWait LoopAction = iota

	// LoopContinue starts another model turn without waiting for new input.
	LoopContinue

	// LoopStop closes the session in both modes: a started session stops
	// serving, and a session that returned from Run cannot be Run again.
	LoopStop
)

func ReactLoop

func ReactLoop(_ context.Context, turn *SessionTurnResult) (LoopAction, error)

ReactLoop continues while the last turn produced a tool result, which is the plain reason-act cycle: the model called tools, so it must see their results.

type Model

type Model struct {
	Name     string
	Model    string
	MaxToken int64
	Provider types.Provider
	Fallback *ModelFallback
}

func GetModel

func GetModel(name string) (*Model, error)

func (*Model) Completion

func (m *Model) Completion(ctx context.Context, messages []*types.Message, options ...types.CompletionOption) (*types.Completion, error)

func (*Model) Embedding

func (m *Model) Embedding(ctx context.Context, input []string, options ...types.EmbeddingOption) (*types.Embedding, error)

func (*Model) Realtime

func (m *Model) Realtime(ctx context.Context, messages []*types.Message, options ...types.RealTimeOption) (types.RealTimeSession, error)

func (*Model) Rerank

func (m *Model) Rerank(ctx context.Context, input *types.RerankInput, options ...types.RerankOption) (*types.RerankOutput, error)

type ModelConfig

type ModelConfig struct {
	Name     string               `yaml:"name"`
	Model    string               `yaml:"model"`
	Provider string               `yaml:"provider"`
	MaxToken int64                `yaml:"maxToken"`
	Fallback *ModelConfigFallback `yaml:"fallback"`
}

type ModelConfigFallback

type ModelConfigFallback struct {
	MaxSeconds int                           `yaml:"maxSeconds"`
	Providers  []ModelConfigFallbackProvider `yaml:"providers"`
}

type ModelConfigFallbackProvider

type ModelConfigFallbackProvider struct {
	Provider string `yaml:"provider"`
	Model    string `yaml:"model"`
}

type ModelFallback

type ModelFallback struct {
	MaxSeconds int
	Providers  []*ModelFallbackProvider
	// contains filtered or unexported fields
}

type ModelFallbackProvider

type ModelFallbackProvider struct {
	ProviderName string
	Provider     types.Provider
	Model        string
}

type ModelInfo

type ModelInfo struct {
	MaxToken       int64 `yaml:"maxToken"`
	MaxOutputToken int64 `yaml:"maxOutputToken"`
}

type Models

type Models struct {
	// contains filtered or unexported fields
}
var DefaultModels *Models = NewModels()

func NewModels

func NewModels() *Models

func NewModelsWithConfig

func NewModelsWithConfig(config ModelsConfig) (*Models, error)

func NewModelsWithConfigFile

func NewModelsWithConfigFile(fn string) (*Models, error)

func (*Models) AddModel

func (m *Models) AddModel(conf ModelConfig) (*Model, error)

func (*Models) AddProvider

func (m *Models) AddProvider(conf ProviderConfig) (types.Provider, error)

func (*Models) Close

func (m *Models) Close() error

func (*Models) Completion

func (ms *Models) Completion(ctx context.Context, model string, messages []*types.Message, options ...types.CompletionOption) (*types.Completion, error)

func (*Models) Embedding

func (ms *Models) Embedding(ctx context.Context, model string, input []string, options ...types.EmbeddingOption) (*types.Embedding, error)

func (*Models) GetModel

func (m *Models) GetModel(name string) (*Model, error)

func (*Models) Realtime

func (ms *Models) Realtime(ctx context.Context, model string, messages []*types.Message, options ...types.RealTimeOption) (types.RealTimeSession, error)

func (*Models) RegisterProvider

func (m *Models) RegisterProvider(name string, provider types.Provider) error

func (*Models) Rerank

func (ms *Models) Rerank(ctx context.Context, model string, input *types.RerankInput, options ...types.RerankOption) (*types.RerankOutput, error)

type ModelsConfig

type ModelsConfig struct {
	Providers []ProviderConfig `yaml:"providers"`
	Models    []ModelConfig    `yaml:"models"`
}

type ProviderConfig

type ProviderConfig struct {
	Name          string                       `yaml:"name"`
	Provider      string                       `yaml:"provider"`
	Url           string                       `yaml:"url"`
	Insecure      bool                         `yaml:"insecure"`
	Sk            string                       `yaml:"sk"`
	ThinkArgs     *types.ProviderThinkArgs     `yaml:"thinkArgs"`
	EmbeddingArgs *types.ProviderEmbeddingArgs `yaml:"embeddingArgs"`
}

type ProviderCreator

type ProviderCreator func(opts ...types.ProviderOption) (types.Provider, error)

type ProviderEnv

type ProviderEnv struct {
	Provider     string
	Url          string
	LegacyURL    string
	ApiKey       string
	LegacyAPIKey string
}

type Runner

type Runner struct {
	// contains filtered or unexported fields
}

Runner is the asynchronous handle for a session started with Start.

func (*Runner) Done

func (r *Runner) Done() <-chan struct{}

Done is closed when the started session finishes.

func (*Runner) Send

func (r *Runner) Send(ctx context.Context, input *SessionInput) error

Send submits one input. It returns once the session accepts ownership of the input; processing continues under the context passed to Session.Start.

A session only accepts input while it is waiting, so Send blocks for as long as the current run of autonomous turns lasts. Pass a ctx with a deadline to bound the wait; a Send the session never accepted fails with that ctx error or with ErrSessionClosed, never silently.

func (*Runner) Wait

func (r *Runner) Wait() (*SessionResult, error)

Wait blocks until the started session finishes and returns its final snapshot.

type Session

type Session struct {
	// contains filtered or unexported fields
}

Session owns conversation history, state, and tool correlation across runs. Run processes one fixed input synchronously. Start switches the session into a long-running mode that waits for inputs until its context is canceled.

func NewSession

func NewSession(
	agent *Agent,
	history []*Event,
	opts ...AgentOption,
) (*Session, error)

NewSession restores one reusable in-memory session. Historical events and their nested values are treated as immutable and retained by reference; clone them before making changes. They restore state but are neither re-emitted to EventSink nor included in future results.

func (*Session) Run

func (s *Session) Run(
	ctx context.Context,
	input *SessionInput,
) (*SessionResult, error)

Run processes one fixed input and every autonomous turn it causes. It returns when Loop yields or stops. After LoopWait the session can be Run again with a later input. Any error -- including cancellation of this Run's context -- closes the session, because a partial turn may leave the history in a shape the model must not resume; rebuild a session from the sinked events instead.

func (*Session) Start

func (s *Session) Start(ctx context.Context) (*Runner, error)

Start switches this session into long-running mode. It initially waits for input, runs autonomous turns while Loop returns LoopContinue, and returns to waiting whenever Loop returns LoopWait. The context owns the whole lifetime.

type SessionHumanResponse

type SessionHumanResponse struct {
	ID     string         `json:"id,omitempty" yaml:"id,omitempty"`
	Result map[string]any `json:"result,omitempty" yaml:"result,omitempty"`
}

SessionHumanResponse answers one human request raised by a session. ID must match the ID of an EventHumanRequest emitted by the session.

type SessionInput

type SessionInput struct {
	// Messages are appended to the model history, one message event each.
	Messages []*types.Message `json:"messages,omitempty" yaml:"messages,omitempty"`

	// HumanResponses resolve human requests that are still pending.
	HumanResponses []*SessionHumanResponse `json:"humanResponses,omitempty" yaml:"humanResponses,omitempty"`

	// State replaces the session state before the rest of this input.
	// StateDelta merges into the current session state instead.
	State      map[string]string `json:"state,omitempty" yaml:"state,omitempty"`
	StateDelta map[string]string `json:"stateDelta,omitempty" yaml:"stateDelta,omitempty"`
}

SessionInput is the only way a caller feeds new input into a session. It is used both by Session.Run and as the payload of Runner.Send.

Callers never build input events themselves: the session turns one input into events, so every event a caller observes comes from session output.

func NewHumanResponseInput

func NewHumanResponseInput(
	requestID string,
	result map[string]any,
) *SessionInput

NewHumanResponseInput creates one input answering a pending human request.

func NewSessionInput

func NewSessionInput(messages ...*types.Message) *SessionInput

NewSessionInput creates one input carrying messages.

func NewUserInput

func NewUserInput(text string) *SessionInput

NewUserInput creates one input carrying a single user text message.

func (*SessionInput) Clone

func (i *SessionInput) Clone() *SessionInput

func (*SessionInput) IsEmpty

func (i *SessionInput) IsEmpty() bool

IsEmpty reports whether this input carries nothing for the session to accept.

func (*SessionInput) Validate

func (i *SessionInput) Validate() error

func (*SessionInput) WithHumanResponse

func (i *SessionInput) WithHumanResponse(
	id string,
	result map[string]any,
) *SessionInput

WithHumanResponse appends one human response to this input.

func (*SessionInput) WithMessages

func (i *SessionInput) WithMessages(messages ...*types.Message) *SessionInput

WithMessages appends messages to this input.

func (*SessionInput) WithState

func (i *SessionInput) WithState(state map[string]string, delta bool) *SessionInput

WithState sets either a full state replacement or a state patch on this input. When delta is true, state is merged; otherwise it replaces the state.

type SessionResult

type SessionResult struct {
	Events []*Event
	State  map[string]string

	Turns      int
	StopReason StopReason
	Cause      error
}

SessionResult is the final snapshot produced by one Run, or by the lifetime of a session started with Start.

func (*SessionResult) Clone

func (r *SessionResult) Clone() *SessionResult

type SessionTurn

type SessionTurn struct {
	Index       int
	Model       string
	Instruction string
	History     []*types.Message
	Tools       []*Tool
	// contains filtered or unexported fields
}

SessionTurn is the immutable input for one model turn in a session. History, tools, and their nested values are read-only; clone the turn before changing its slice containers. State is exposed only through read-only snapshots.

func (*SessionTurn) Clone

func (t *SessionTurn) Clone() *SessionTurn

Clone returns a turn with independently owned slice containers and state. Messages and tools remain shared immutable values.

func (*SessionTurn) GetState

func (t *SessionTurn) GetState(key string) (string, bool)

GetState reads one value from the state snapshot captured for this turn.

func (*SessionTurn) StateSnapshot

func (t *SessionTurn) StateSnapshot() map[string]string

StateSnapshot returns an independently owned copy of this turn's state. Changing the returned map never changes the session.

type SessionTurnResult

type SessionTurnResult struct {
	Index          int
	Events         []*Event
	TurnEventIndex int
}

SessionTurnResult is what one model turn produced, after every event of that turn is committed. It is the input a Loop decides on.

type StopReason

type StopReason string
const (
	StopDone     StopReason = "done"
	StopMaxTurn  StopReason = "maxTurn"
	StopCanceled StopReason = "canceled"
	StopError    StopReason = "error"
)

type Tool

type Tool struct {
	Name        string
	Description string
	Parameters  map[string]any
	Invoker     ToolInvoker
}

func MustNewTool

func MustNewTool[T any](name string, desc string, invoker func(ToolContext, *T) (any, error)) *Tool

func NewTool

func NewTool[T any](name string, desc string, invoker func(ToolContext, *T) (any, error)) (*Tool, error)

type ToolCallHandler

type ToolCallHandler func(ToolContext, *types.MessageToolCall) (any, error)

type ToolCallInterceptor

type ToolCallInterceptor func(ToolCallHandler) ToolCallHandler

type ToolContext

type ToolContext interface {
	context.Context
	ToolCallID() string
	State() map[string]string
	SetState(patch map[string]string) error
	// HumanRequest and HumanResponse return immutable session values. Call Clone
	// before modifying nested fields.
	HumanRequest() *EventHumanRequest
	HumanResponse() *EventHumanResponse
}

type ToolInvoker

type ToolInvoker func(ToolContext, string) (any, error)

type ToolResultHuman

type ToolResultHuman struct {
	Arguments map[string]any
}

type ToolResultNotify

type ToolResultNotify struct{}

type ToolResultWithInstruction

type ToolResultWithInstruction struct {
	Result      any
	Instruction map[string]string
}

type ToolRun

type ToolRun struct {
	Call         *types.MessageToolCall
	Result       any
	Error        error
	Duration     time.Duration
	StatePatches []map[string]string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL