Documentation
¶
Index ¶
- type IS
- type ModelClient
- type NoopModelClient
- type NoopRewriter
- type NoopVectorStoreClient
- type Rewriter
- type S
- func (s *S) CreateChatCompletion(w http.ResponseWriter, req *http.Request, pathParams map[string]string)
- func (s *S) CreateCompletion(w http.ResponseWriter, req *http.Request, pathParams map[string]string)
- func (s *S) CreateEmbedding(w http.ResponseWriter, req *http.Request, pathParams map[string]string)
- func (s *S) Run(ctx context.Context, port int, authConfig config.AuthConfig) error
- func (s *S) Stop()
- type VectorStoreClient
- type WS
- func (ws *WS) ProcessTasks(srv v1.InferenceWorkerService_ProcessTasksServer) error
- func (ws *WS) Run(ctx context.Context, port int, authConfig config.AuthConfig, ...) error
- func (ws *WS) RunWithListener(ctx context.Context, authConfig config.AuthConfig, tlsConfig *config.TLS, ...) error
- func (ws *WS) Stop()
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type IS ¶ added in v1.4.0
type IS struct {
v1.UnimplementedInferenceInternalServiceServer
// contains filtered or unexported fields
}
IS is a server for internal services.
func NewInternalServer ¶ added in v1.4.0
func NewInternalServer( infProcessor *infprocessor.P, taskExchanger *taskexchanger.E, logger logr.Logger, ) *IS
NewInternalServer creates a new internal server.
func (*IS) ProcessTasksInternal ¶ added in v1.4.0
func (is *IS) ProcessTasksInternal(srv v1.InferenceInternalService_ProcessTasksInternalServer) error
ProcessTasksInternal processes tasks.
func (*IS) RunWithListener ¶ added in v1.4.0
RunWithListener runs the server with a given listener.
type ModelClient ¶
type ModelClient interface {
GetModel(ctx context.Context, in *mv1.GetModelRequest, opts ...grpc.CallOption) (*mv1.Model, error)
}
ModelClient is an interface for a model client.
type NoopModelClient ¶
type NoopModelClient struct {
}
NoopModelClient is a no-op model client.
func (*NoopModelClient) GetModel ¶
func (c *NoopModelClient) GetModel(ctx context.Context, in *mv1.GetModelRequest, opts ...grpc.CallOption) (*mv1.Model, error)
GetModel is a no-op implementation of GetModel.
type NoopRewriter ¶
type NoopRewriter struct {
}
NoopRewriter is a no-op rewriter.
func (*NoopRewriter) ProcessMessages ¶
func (r *NoopRewriter) ProcessMessages( ctx context.Context, vstore *vsv1.VectorStore, messages []*v1.CreateChatCompletionRequest_Message, ) ([]*v1.CreateChatCompletionRequest_Message, error)
ProcessMessages is a no-op implementation of ProcessMessages.
type NoopVectorStoreClient ¶
type NoopVectorStoreClient struct {
}
NoopVectorStoreClient is a no-op vector store client.
func (*NoopVectorStoreClient) GetVectorStoreByName ¶
func (c *NoopVectorStoreClient) GetVectorStoreByName(ctx context.Context, req *vsv1.GetVectorStoreByNameRequest, opts ...grpc.CallOption) (*vsv1.VectorStore, error)
GetVectorStoreByName is a no-op implementation of GetVectorStoreByName.
type Rewriter ¶
type Rewriter interface {
ProcessMessages(
ctx context.Context,
vstore *vsv1.VectorStore,
messages []*v1.CreateChatCompletionRequest_Message,
) ([]*v1.CreateChatCompletionRequest_Message, error)
}
Rewriter is an interface for rag.
type S ¶
type S struct {
v1.UnimplementedChatServiceServer
// contains filtered or unexported fields
}
S is a server.
func New ¶
func New( m metricsMonitoring, usage sender.UsageSetter, rate rate.Limiter, modelClient ModelClient, vsClient VectorStoreClient, r Rewriter, taskSender taskSender, logger logr.Logger, ) *S
New creates a server.
func (*S) CreateChatCompletion ¶
func (s *S) CreateChatCompletion( w http.ResponseWriter, req *http.Request, pathParams map[string]string, )
CreateChatCompletion creates a chat completion.
func (*S) CreateCompletion ¶
func (s *S) CreateCompletion( w http.ResponseWriter, req *http.Request, pathParams map[string]string, )
CreateCompletion creates a (legacy) completion.
The implementation is similar to CreateChatCompletion, but this has extra logic for converting a legacy request to a non-legacy request (and vice versa for response).
TODO(kenji): Avoid code duplication CreateChatCompletion.
func (*S) CreateEmbedding ¶
func (s *S) CreateEmbedding( w http.ResponseWriter, req *http.Request, pathParams map[string]string, )
CreateEmbedding creates an embedding.
type VectorStoreClient ¶
type VectorStoreClient interface {
GetVectorStoreByName(ctx context.Context, req *vsv1.GetVectorStoreByNameRequest, opts ...grpc.CallOption) (*vsv1.VectorStore, error)
}
VectorStoreClient is an interface for a vector store client.
type WS ¶
type WS struct {
v1.UnimplementedInferenceWorkerServiceServer
// contains filtered or unexported fields
}
WS is a server for worker services.
func NewWorkerServiceServer ¶
func NewWorkerServiceServer(infProcessor *infprocessor.P, logger logr.Logger) *WS
NewWorkerServiceServer creates a new worker service server.
func (*WS) ProcessTasks ¶
func (ws *WS) ProcessTasks(srv v1.InferenceWorkerService_ProcessTasksServer) error
ProcessTasks processes tasks.
func (*WS) Run ¶
func (ws *WS) Run(ctx context.Context, port int, authConfig config.AuthConfig, tlsConfig *config.TLS) error
Run runs the worker service server.
Source Files