Documentation
¶
Overview ¶
Package vllm provides vLLM OpenAI-compatible API integration for the LLM gateway.
Package vllm provides vLLM OpenAI-compatible API integration for the LLM gateway.
Index ¶
- Variables
- func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider
- type Provider
- func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (*core.ChatResponse, error)
- func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) (*core.EmbeddingResponse, error)
- func (p *Provider) ListModels(ctx context.Context) (*core.ModelsResponse, error)
- func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest) (*core.PassthroughResponse, error)
- func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error)
- func (p *Provider) SetBaseURL(url string)
- func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatRequest) (io.ReadCloser, error)
- func (p *Provider) StreamResponses(ctx context.Context, req *core.ResponsesRequest) (io.ReadCloser, error)
Constants ¶
This section is empty.
Variables ¶
var Registration = providers.Registration{ Type: "vllm", New: New, PassthroughSemanticEnricher: passthroughSemanticEnricher, Discovery: providers.DiscoveryConfig{ DefaultBaseURL: defaultBaseURL, AllowAPIKeyless: true, }, }
Registration provides factory registration for the vLLM provider.
Functions ¶
func New ¶
func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider
New creates a new vLLM provider.
Types ¶
type Provider ¶
type Provider struct {
// contains filtered or unexported fields
}
Provider implements the core.Provider interface for vLLM.
func (*Provider) ChatCompletion ¶
func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (*core.ChatResponse, error)
ChatCompletion sends a chat completion request to vLLM.
func (*Provider) Embeddings ¶
func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) (*core.EmbeddingResponse, error)
Embeddings sends an embeddings request to vLLM.
func (*Provider) ListModels ¶
ListModels retrieves the list of available models from vLLM. ListModels lists the served models, keeping the max_model_len vLLM reports per entry as the context window.
func (*Provider) Passthrough ¶
func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest) (*core.PassthroughResponse, error)
Passthrough routes an opaque provider-native request to vLLM.
func (*Provider) Responses ¶
func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error)
Responses sends a Responses API request to vLLM.
func (*Provider) SetBaseURL ¶
SetBaseURL allows configuring a custom base URL for the provider.
func (*Provider) StreamChatCompletion ¶
func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatRequest) (io.ReadCloser, error)
StreamChatCompletion returns a raw response body for streaming.
func (*Provider) StreamResponses ¶
func (p *Provider) StreamResponses(ctx context.Context, req *core.ResponsesRequest) (io.ReadCloser, error)
StreamResponses streams a Responses API request to vLLM.