Documentation
¶
Overview ¶
Package vllm provides vLLM LLM provider integration for high-performance inference.
Package vllm provides multimodal support for vLLM provider
Index ¶
- type Provider
- func (p *Provider) BuildTooling(descriptors []*providers.ToolDescriptor) (any, error)
- func (p *Provider) CalculateCost(tokensIn, tokensOut, cachedTokens int) types.CostInfo
- func (p *Provider) GetMultimodalCapabilities() providers.MultimodalCapabilities
- func (p *Provider) Model() string
- func (p *Provider) Predict(ctx context.Context, req providers.PredictionRequest) (providers.PredictionResponse, error)
- func (p *Provider) PredictMultimodal(ctx context.Context, req providers.PredictionRequest) (providers.PredictionResponse, error)
- func (p *Provider) PredictMultimodalStream(ctx context.Context, req providers.PredictionRequest) (<-chan providers.StreamChunk, error)
- func (p *Provider) PredictStream(ctx context.Context, req providers.PredictionRequest) (<-chan providers.StreamChunk, error)
- func (p *Provider) PredictStreamWithTools(ctx context.Context, req providers.PredictionRequest, tools any, ...) (<-chan providers.StreamChunk, error)
- func (p *Provider) PredictWithTools(ctx context.Context, req providers.PredictionRequest, tools any, ...) (providers.PredictionResponse, []types.MessageToolCall, error)
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Provider ¶
type Provider struct {
providers.BaseProvider
// contains filtered or unexported fields
}
Provider implements the Provider interface for vLLM
func NewProvider ¶
func NewProvider( id, model, baseURL string, defaults providers.ProviderDefaults, includeRawOutput bool, additionalConfig map[string]any, ) *Provider
NewProvider creates a new vLLM provider
func (*Provider) BuildTooling ¶
func (p *Provider) BuildTooling(descriptors []*providers.ToolDescriptor) (any, error)
BuildTooling converts tool descriptors to vLLM format
func (*Provider) CalculateCost ¶
CalculateCost calculates cost breakdown vLLM is typically self-hosted, so default is $0 unless custom pricing is configured
func (*Provider) GetMultimodalCapabilities ¶
func (p *Provider) GetMultimodalCapabilities() providers.MultimodalCapabilities
GetMultimodalCapabilities returns the multimodal capabilities of the vLLM provider
func (*Provider) Predict ¶
func (p *Provider) Predict( ctx context.Context, req providers.PredictionRequest, ) (providers.PredictionResponse, error)
Predict sends a prediction request to vLLM
func (*Provider) PredictMultimodal ¶
func (p *Provider) PredictMultimodal( ctx context.Context, req providers.PredictionRequest, ) (providers.PredictionResponse, error)
PredictMultimodal sends a multimodal prediction request to vLLM vLLM supports vision models via OpenAI-compatible API with image_url format
func (*Provider) PredictMultimodalStream ¶
func (p *Provider) PredictMultimodalStream( ctx context.Context, req providers.PredictionRequest, ) (<-chan providers.StreamChunk, error)
PredictMultimodalStream sends a streaming multimodal prediction request to vLLM
func (*Provider) PredictStream ¶
func (p *Provider) PredictStream( ctx context.Context, req providers.PredictionRequest, ) (<-chan providers.StreamChunk, error)
PredictStream streams a prediction response from vLLM
func (*Provider) PredictStreamWithTools ¶
func (p *Provider) PredictStreamWithTools( ctx context.Context, req providers.PredictionRequest, tools any, toolChoice string, ) (<-chan providers.StreamChunk, error)
PredictStreamWithTools performs a streaming prediction request with tool support
func (*Provider) PredictWithTools ¶
func (p *Provider) PredictWithTools( ctx context.Context, req providers.PredictionRequest, tools any, toolChoice string, ) (providers.PredictionResponse, []types.MessageToolCall, error)
PredictWithTools performs a prediction request with tool support
Source Files