Documentation
¶
Overview ¶
Package mllm is simple llm client, part of aerth/mostly project
Index ¶
- Variables
- func StderrHandler(cr ChatResponse) error
- func StdoutHandler(cr ChatResponse) error
- func Summary(w io.Writer, st Stats) (err error)
- type ChatHandlerT
- type ChatMessage
- type ChatMessageFormatter
- type ChatMessages
- type ChatRequest
- type ChatResponse
- type E
- func (ee E) AddHandlers(h ...StreamHandler) E
- func (ee E) Get(ctx context.Context, messages []Message, think any) (ChatResponse, error)
- func (ee E) Stream(ctx context.Context, messages []Message, think any, handlers ...StreamHandler) error
- func (ee E) StreamBG(ctx context.Context, messages []Message, think any, handlers ...StreamHandler) chan error
- func (ee E) ValidateConfig(think any) error
- func (ee E) WithHandlers(h ...StreamHandler) E
- func (ee E) WithOptions(options *Options) E
- func (ee E) WithTools(tools ...json.Marshaler) E
- func (ee E) WithoutTools() E
- type Message
- type Messages
- type Options
- type Stats
- type StreamHandler
- type Think
Constants ¶
This section is empty.
Variables ¶
var ( DefaultThinkPre = "Thinking:\n" DefaultThinkPost = "\n\n... done thinking.\n\n" )
var ( EnvModel = []string{"MLLM_MODEL", "OLLAMA_MODEL", "LLM_MODEL"} EnvHost = []string{"MLLM_HOST", "OLLAMA_HOST", "LLM_HOST"} )
var ( StopAtFirstHandlerError = true ErrNoLines = errors.New("no lines read") )
var ( Errorf = stackerr.Errorf Slog = slog.Default )
var Default = func() E { e := NewE(nil, nil) return e }
Default creates a new reusable LLM endpoint from env using NewEndpoint with no tools no options.
var DefaultChatMessageFormatter = func(cm ChatMessage) ([]Message, bool) { return []Message{cm.Message}, true }
DefaultChatMessageFormatter pass through ChatMessageFormatter In a custom implementation, this might add text to Content, replace Role, or insert messages before/after.
var DefaultWriter io.Writer = nil
DefaultWriter added to all new endpoints
var NewE = NewEndpoint
NewE custom NewEndpoint
Functions ¶
Types ¶
type ChatHandlerT ¶
type ChatHandlerT interface {
HandleChat(cr ChatResponse) error
}
type ChatMessage ¶
type ChatMessage struct {
UUID uuid.UUID `json:"uuid,omitempty" gorm:"uniqueIndex"`
Message
From string `json:"from,omitempty" gorm:"default:null"`
CreatedAt time.Time `json:"created_at"`
}
ChatMessage is a Message but with additional metadata (for a multi-participant conversation)
func NewChatMessage ¶
func NewChatMessage(from string, role string, text string) ChatMessage
NewChatMessage from.
type ChatMessageFormatter ¶
type ChatMessageFormatter = func(cm ChatMessage) ([]Message, bool)
ChatMessageFormatter may skip a message by returning false
The returned formatted messages are NEVER stored only sent
type ChatMessages ¶
type ChatMessages []ChatMessage
func (ChatMessages) ToMessages ¶
func (cm ChatMessages) ToMessages() []Message
func (ChatMessages) ToMessagesFormat ¶
func (cm ChatMessages) ToMessagesFormat(fn ChatMessageFormatter) []Message
ToMessagesFormat returns a **temporary** convo for llm to produce a single new Message The returned convo should NEVER be stored (only sent to llm) as the Content is modified.
type ChatRequest ¶
type ChatRequest struct {
Model string `json:"model,omitempty"`
Messages []Message `json:"messages,omitempty"`
Stream bool `json:"stream"`
Think *Think `json:"think,omitempty"`
KeepAlive int `json:"keep_alive,omitempty"`
Tools []json.Marshaler `json:"tools,omitempty"`
Options *Options `json:"options,omitempty"`
}
ChatRequest sent to LLM
type ChatResponse ¶
type ChatResponse struct {
Model string `json:"model"`
Created json.RawMessage `json:"created_at,omitempty"`
Done bool `json:"done"` // for stream
DoneReason string `json:"done_reason,omitempty"`
Message Message `json:"message"` // for stream, handlers
Stats `gorm:"embedded"`
}
ChatResponse streamable has ChatResponse.Handler
Example:
var cr mllm.ChatResponse
err := <-mllm.Default().StreamBG(ctx, mllm.Messages{{Role: "user", Content: "hi"}}, false, cr.Handler)
println("Bot:", cr.Message.Content)
func (*ChatResponse) Handler ¶
func (c *ChatResponse) Handler(cr ChatResponse) error
func (ChatResponse) SummaryString ¶
func (c ChatResponse) SummaryString() string
SummaryString collects [ChatResponse.Statis] into a string
type E ¶
type E struct {
Host url.URL // the LLM API (required)
Model string // the model name (required)
Client *http.Client // custom http client (optional)
Headers map[string]string // custom headers (optional)
Tools []json.Marshaler // Bring your own tools (optional)
Options *Options // e options (optional)
// Writer if non-nil will be automatically added as a stream handler to [[E.Get]] and [[E.Stream]]
Writer io.Writer
ThinkPre string // optional, default used in [NewEndpoint]
ThinkPost string // optional, default used in [NewEndpoint]
// Handlers added to all requests
Handlers []StreamHandler
}
E (Endpoint) holds information for future requests
Writer for streaming output with ThinkPre and ThinkPost.
func NewEndpoint ¶
NewEndpoint reads env and returns a new reusable Endpoint
See E.StreamBG or E.Stream or E.Get
func (E) AddHandlers ¶
func (ee E) AddHandlers(h ...StreamHandler) E
func (E) StreamBG ¶
func (ee E) StreamBG(ctx context.Context, messages []Message, think any, handlers ...StreamHandler) chan error
StreamBG uses Stream, closing ch after streaming is finished, sending non-nil error to the channel
func (E) ValidateConfig ¶
func (E) WithHandlers ¶
func (ee E) WithHandlers(h ...StreamHandler) E
func (E) WithOptions ¶
func (E) WithoutTools ¶ added in v0.2.1
type Message ¶
type Message struct {
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
Reasoning string `json:"thinking,omitempty" gorm:"default:default:null"`
ToolCalls []json.RawMessage `json:"tool_calls,omitempty" gorm:"type:jsonb;serializer:json;default:null;"`
ToolName string `json:"tool_name,omitempty" gorm:"default:null"`
ToolCallID string `json:"tool_call_id,omitempty" gorm:"default:null"`
Images [][]byte `json:"images,omitempty" gorm:"serializer:json;type:jsonb;default:null;"`
}
func (Message) ToChatMessage ¶
func (m Message) ToChatMessage(from string) ChatMessage
ToChatMessage with new UUID,from,timestamp. should be called immediately after stream is finished..
ChatMessage MAY be stored (NOT plain Message)
type Options ¶
type Options struct {
Seed int64 `json:"seed,omitempty"`
Temperature *float64 `json:"temperature,omitempty"` // 0.0 to 2.0
NumCtx int `json:"num_ctx,omitempty"`
TopP *float64 `json:"top_p,omitempty"` // 0.0 to 1.0
TopK *int `json:"top_k,omitempty"` // 1 to ???
FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` // -2.0 to 2.0
PresencePenalty *float64 `json:"presence_penalty,omitempty"` // -2.0 to 2.0
//
Debug bool `json:"-"` // internal, for printing full chatrequest before sending
}
Options for ChatRequest
type Stats ¶
type Stats struct {
LoadDuration time.Duration `json:"load_duration,omitempty"`
PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
EvalDuration time.Duration `json:"eval_duration,omitempty"`
TotalDuration time.Duration `json:"total_duration,omitempty"`
PromptEvalCount int `json:"prompt_eval_count,omitempty"`
EvalCount int `json:"eval_count,omitempty"`
}
Stats has [Stats.String]
func (Stats) SummaryString ¶ added in v0.2.1
SummaryString collects [ChatResponse.Summary] into a string
type StreamHandler ¶
type StreamHandler func(cr ChatResponse) error
func NewHandlerOut ¶
func NewHandlerOut(w io.Writer, thinkpre string, thinkpost string) StreamHandler
NewHandlerOut plain 🍔
func (StreamHandler) HandleChat ¶
func (c StreamHandler) HandleChat(cr ChatResponse) error
type Think ¶
type Think struct {
Value any
}
Think Value should be one of: nil,"low","medium","high",true,false
func (Think) MarshalJSON ¶
MarshalJSON implements json.Marshaler for sending request
Source Files