Documentation
¶
Overview ¶
Package vllm implements the modelrepo.Provider contract against vLLM OpenAI-compatible HTTP endpoints. The package registers its catalog at init time; depend on it via blank import where the catalog must be discoverable from runtimestate.
Index ¶
- func NewVLLMChatClient(ctx context.Context, baseURL, modelName string, ...) (modelrepo.LLMChatClient, error)
- func NewVLLMPromptClient(ctx context.Context, baseURL, modelName string, ...) (modelrepo.LLMPromptExecClient, error)
- func NewVLLMProvider(modelName string, backends []string, client *http.Client, ...) modelrepo.Provider
- func NewVLLMStreamClient(ctx context.Context, baseURL, modelName string, ...) (modelrepo.LLMStreamClient, error)
- type VLLMChatClient
- type VLLMStreamClient
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func NewVLLMChatClient ¶
func NewVLLMChatClient(ctx context.Context, baseURL, modelName string, contextLength, maxOutputTokens int, httpClient *http.Client, apiKey string, canThink bool, tracker libtracker.ActivityTracker) (modelrepo.LLMChatClient, error)
func NewVLLMPromptClient ¶
func NewVLLMPromptClient(ctx context.Context, baseURL, modelName string, contextLength, maxOutputTokens int, httpClient *http.Client, apiKey string, canThink bool, tracker libtracker.ActivityTracker) (modelrepo.LLMPromptExecClient, error)
NewVLLMPromptClient creates a new prompt client
func NewVLLMProvider ¶
func NewVLLMProvider(modelName string, backends []string, client *http.Client, caps modelrepo.CapabilityConfig, authToken string, tracker libtracker.ActivityTracker) modelrepo.Provider
func NewVLLMStreamClient ¶
func NewVLLMStreamClient(ctx context.Context, baseURL, modelName string, contextLength, maxOutputTokens int, httpClient *http.Client, apiKey string, canThink bool, tracker libtracker.ActivityTracker) (modelrepo.LLMStreamClient, error)
Types ¶
type VLLMChatClient ¶
type VLLMChatClient struct {
// contains filtered or unexported fields
}
func (*VLLMChatClient) Chat ¶
func (c *VLLMChatClient) Chat(ctx context.Context, messages []modelrepo.Message, args ...modelrepo.ChatArgument) (modelrepo.ChatResult, error)
type VLLMStreamClient ¶
type VLLMStreamClient struct {
// contains filtered or unexported fields
}
func (*VLLMStreamClient) Prompt ¶
func (c *VLLMStreamClient) Prompt(ctx context.Context, systemInstruction string, temperature float32, prompt string) (string, error)
Prompt implements LLMPromptExecClient interface
func (*VLLMStreamClient) Stream ¶
func (c *VLLMStreamClient) Stream(ctx context.Context, messages []modelrepo.Message, args ...modelrepo.ChatArgument) (<-chan *modelrepo.StreamParcel, error)
Stream implements LLMStreamClient interface
Source Files
Click to show internal directories.
Click to hide internal directories.