Documentation
¶
Index ¶
- Constants
- Variables
- func GetModelTag(model string) string
- func GetTraceID(traceparent, requestID string) string
- type MockCache
- func (m *MockCache) AddRequestCount(ctx *types.RoutingContext, requestID string, model string) int64
- func (m *MockCache) AddSubscriber(subscriber metrics.MetricSubscriber)
- func (m *MockCache) DoneRequestCount(ctx *types.RoutingContext, requestID string, model string, term int64)
- func (m *MockCache) DoneRequestTrace(ctx *types.RoutingContext, requestID string, model string, inputTokens int64, ...)
- func (m *MockCache) GetMetricValueByPod(namespace string, podName string, metricName string) (metrics.MetricValue, error)
- func (m *MockCache) GetMetricValueByPodModel(namespace string, podName string, model string, metricName string) (metrics.MetricValue, error)
- func (m *MockCache) GetPod(namespace string, podName string) (*v1.Pod, error)
- func (m *MockCache) HasModel(model string) bool
- func (m *MockCache) ListModels() []string
- func (m *MockCache) ListModelsByPod(namespace string, podName string) ([]string, error)
- func (m *MockCache) ListPodsByModel(model string) (types.PodList, error)
- type MockGatewayClassClient
- type MockGatewayClient
- func (m *MockGatewayClient) Discovery() discovery.DiscoveryInterface
- func (m *MockGatewayClient) GatewayV1() gatewayapiv1.GatewayV1Interface
- func (m *MockGatewayClient) GatewayV1alpha2() gatewayapiv1alpha2.GatewayV1alpha2Interface
- func (m *MockGatewayClient) GatewayV1beta1() gatewayapiv1beta1.GatewayV1beta1Interface
- type MockGatewayV1Client
- func (m *MockGatewayV1Client) GatewayClasses() gatewayapiv1.GatewayClassInterface
- func (m *MockGatewayV1Client) Gateways(namespace string) gatewayapiv1.GatewayInterface
- func (m *MockGatewayV1Client) HTTPRoutes(namespace string) gatewayapiv1.HTTPRouteInterface
- func (m *MockGatewayV1Client) RESTClient() rest.Interface
- type MockHTTPRouteClient
- func (m *MockHTTPRouteClient) Create(ctx context.Context, httpRoute *gatewayv1.HTTPRoute, opts metav1.CreateOptions) (*gatewayv1.HTTPRoute, error)
- func (m *MockHTTPRouteClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
- func (m *MockHTTPRouteClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
- func (m *MockHTTPRouteClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*gatewayv1.HTTPRoute, error)
- func (m *MockHTTPRouteClient) List(ctx context.Context, opts metav1.ListOptions) (*gatewayv1.HTTPRouteList, error)
- func (m *MockHTTPRouteClient) Patch(ctx context.Context, name string, pt k8stype.PatchType, data []byte, ...) (result *gatewayv1.HTTPRoute, err error)
- func (m *MockHTTPRouteClient) Update(ctx context.Context, httpRoute *gatewayv1.HTTPRoute, opts metav1.UpdateOptions) (*gatewayv1.HTTPRoute, error)
- func (m *MockHTTPRouteClient) UpdateStatus(ctx context.Context, httpRoute *gatewayv1.HTTPRoute, opts metav1.UpdateOptions) (*gatewayv1.HTTPRoute, error)
- func (m *MockHTTPRouteClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
- type OpenAIResponse
- type Server
- func (s *Server) HandleRequestBody(ctx context.Context, routingCtx *types.RoutingContext, requestID string, ...) (*extProcPb.ProcessingResponse, string, bool, int64)
- func (s *Server) HandleRequestHeaders(ctx context.Context, requestID string, req *extProcPb.ProcessingRequest) (*extProcPb.ProcessingResponse, utils.User, int64, *types.RoutingContext)
- func (s *Server) HandleResponseBody(ctx context.Context, routerCtx *types.RoutingContext, requestID string, ...) (*extProcPb.ProcessingResponse, bool)
- func (s *Server) HandleResponseHeaders(ctx context.Context, routerCtx *types.RoutingContext, requestID string, ...) (*extProcPb.ProcessingResponse, bool, int)
- func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error
- func (s *Server) Shutdown()
- func (s *Server) StartHTTPServer(addr string) error
Constants ¶
View Source
const ( HeaderErrorInvalidRouting = "x-error-invalid-routing-strategy" // General Error Headers HeaderErrorUser = "x-error-user" HeaderErrorRouting = "x-error-routing" HeaderErrorRequestBodyProcessing = "x-error-request-body-processing" HeaderErrorResponseUnmarshal = "x-error-response-unmarshal" HeaderErrorResponseUnknown = "x-error-response-unknown" // Model & Deployment Headers HeaderErrorNoModelInRequest = "x-error-no-model-in-request" HeaderErrorNoModelBackends = "x-error-no-model-backends" // Streaming Headers HeaderErrorStream = "x-error-stream" HeaderErrorStreaming = "x-error-streaming" HeaderErrorStreamOptionsIncludeUsage = "x-error-no-stream-options-include-usage" // Multipart/Audio Headers HeaderErrorMultipartParsing = "x-error-multipart-parsing" // Request & Target Headers HeaderWentIntoReqHeaders = "x-went-into-req-headers" HeaderTargetPodIP = "target-pod-ip" HeaderTargetPod = "target-pod" HeaderRoutingStrategy = "routing-strategy" HeaderRequestID = "request-id" HeaderModel = "model" HeaderExternalFilter = "external-filter" HeaderConfigProfile = "config-profile" // HeaderSessionID is the header used for session affinity routing. // NOTE: If you change this value, you MUST also update sessionIDHeader in // pkg/plugins/gateway/algorithms/simple_session_affinity.go HeaderSessionID = "x-session-id" HeaderTraceParent = "traceparent" // RPM & TPM Update Errors HeaderUpdateTPM = "x-update-tpm" HeaderUpdateRPM = "x-update-rpm" HeaderErrorRPMExceeded = "x-error-rpm-exceeded" HeaderErrorTPMExceeded = "x-error-tpm-exceeded" HeaderErrorIncrRPM = "x-error-incr-rpm" HeaderErrorIncrTPM = "x-error-incr-tpm" // Model RPS Errors HeaderErrorModelRPSExceeded = "x-error-model-rps-exceeded" HeaderErrorIncrModelRPS = "x-error-incr-model-rps" // Rate Limiting defaults DefaultRPM = 100 DefaultTPMMultiplier = 1000 // Envs EnvRoutingAlgorithm = "ROUTING_ALGORITHM" // OpenAI Error Types ErrorTypeInvalidRequest = "invalid_request_error" ErrorTypeAuthentication = "authentication_error" ErrorTypeRateLimit = "rate_limit_error" ErrorTypeApi = "api_error" ErrorTypeOverloaded = "overloaded_error" // OpenAI Error Codes ErrorCodeInvalidAPIKey = "invalid_api_key" ErrorCodeModelNotFound = "model_not_found" ErrorCodeRateLimitExceeded = "rate_limit_exceeded" // Embedding Constraints // https://github.com/openai/openai-go/blob/main/embedding.go#L126 MaxInputTokensPerModel = 8192 MaxTotalTokens = 300000 MaxArrayDimensions = 2048 // Request Paths PathChatCompletions = "/v1/chat/completions" PathResponses = "/v1/responses" PathMessages = "/v1/messages" PathCompletions = "/v1/completions" PathEmbeddings = "/v1/embeddings" PathImagesGenerations = "/v1/images/generations" PathVideoGenerations = "/v1/video/generations" PathAudioTranscriptions = "/v1/audio/transcriptions" PathAudioTranslations = "/v1/audio/translations" PathRerank = "/v1/rerank" PathClassify = "/v1/classify" // Engine-specific paths (xdit) PathXditGenerate = "/generate" PathXditGenerateVideo = "/generatevideo" // Engine Types EngineXdit = "xdit" )
Variables ¶
View Source
var (
ErrorUnknownResponse = errors.New("unknown response")
)
View Source
var (
POD_NAME = os.Getenv("POD_NAME")
)
Functions ¶
func GetModelTag ¶ added in v0.6.0
func GetTraceID ¶ added in v0.7.0
Types ¶
type MockCache ¶ added in v0.4.0
MockCache implements cache.Cache interface for testing
func (*MockCache) AddRequestCount ¶ added in v0.4.0
func (*MockCache) AddSubscriber ¶ added in v0.4.0
func (m *MockCache) AddSubscriber(subscriber metrics.MetricSubscriber)
func (*MockCache) DoneRequestCount ¶ added in v0.4.0
func (*MockCache) DoneRequestTrace ¶ added in v0.4.0
func (*MockCache) GetMetricValueByPod ¶ added in v0.4.0
func (*MockCache) GetMetricValueByPodModel ¶ added in v0.4.0
func (*MockCache) ListModels ¶ added in v0.4.0
func (*MockCache) ListModelsByPod ¶ added in v0.4.0
type MockGatewayClassClient ¶ added in v0.4.0
type MockGatewayClient ¶ added in v0.4.0
MockGatewayClient implements gatewayapi.Clientset interface
func (*MockGatewayClient) Discovery ¶ added in v0.4.0
func (m *MockGatewayClient) Discovery() discovery.DiscoveryInterface
func (*MockGatewayClient) GatewayV1 ¶ added in v0.4.0
func (m *MockGatewayClient) GatewayV1() gatewayapiv1.GatewayV1Interface
func (*MockGatewayClient) GatewayV1alpha2 ¶ added in v0.4.0
func (m *MockGatewayClient) GatewayV1alpha2() gatewayapiv1alpha2.GatewayV1alpha2Interface
func (*MockGatewayClient) GatewayV1beta1 ¶ added in v0.4.0
func (m *MockGatewayClient) GatewayV1beta1() gatewayapiv1beta1.GatewayV1beta1Interface
type MockGatewayV1Client ¶ added in v0.4.0
MockGatewayV1Client implements gatewayapi.Interface
func (*MockGatewayV1Client) GatewayClasses ¶ added in v0.4.0
func (m *MockGatewayV1Client) GatewayClasses() gatewayapiv1.GatewayClassInterface
func (*MockGatewayV1Client) Gateways ¶ added in v0.4.0
func (m *MockGatewayV1Client) Gateways(namespace string) gatewayapiv1.GatewayInterface
func (*MockGatewayV1Client) HTTPRoutes ¶ added in v0.4.0
func (m *MockGatewayV1Client) HTTPRoutes(namespace string) gatewayapiv1.HTTPRouteInterface
func (*MockGatewayV1Client) RESTClient ¶ added in v0.4.0
func (m *MockGatewayV1Client) RESTClient() rest.Interface
type MockHTTPRouteClient ¶ added in v0.4.0
MockHTTPRouteClient implements gatewayapi.HTTPRouteInterface
func (*MockHTTPRouteClient) Create ¶ added in v0.4.0
func (m *MockHTTPRouteClient) Create(ctx context.Context, httpRoute *gatewayv1.HTTPRoute, opts metav1.CreateOptions) (*gatewayv1.HTTPRoute, error)
func (*MockHTTPRouteClient) Delete ¶ added in v0.4.0
func (m *MockHTTPRouteClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
func (*MockHTTPRouteClient) DeleteCollection ¶ added in v0.4.0
func (m *MockHTTPRouteClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
func (*MockHTTPRouteClient) Get ¶ added in v0.4.0
func (m *MockHTTPRouteClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*gatewayv1.HTTPRoute, error)
func (*MockHTTPRouteClient) List ¶ added in v0.4.0
func (m *MockHTTPRouteClient) List(ctx context.Context, opts metav1.ListOptions) (*gatewayv1.HTTPRouteList, error)
func (*MockHTTPRouteClient) Update ¶ added in v0.4.0
func (m *MockHTTPRouteClient) Update(ctx context.Context, httpRoute *gatewayv1.HTTPRoute, opts metav1.UpdateOptions) (*gatewayv1.HTTPRoute, error)
func (*MockHTTPRouteClient) UpdateStatus ¶ added in v0.4.0
func (m *MockHTTPRouteClient) UpdateStatus(ctx context.Context, httpRoute *gatewayv1.HTTPRoute, opts metav1.UpdateOptions) (*gatewayv1.HTTPRoute, error)
func (*MockHTTPRouteClient) Watch ¶ added in v0.4.0
func (m *MockHTTPRouteClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
type OpenAIResponse ¶ added in v0.5.0
type OpenAIResponse struct {
Model string `json:"model"`
// Usage carries token accounting. The Chat Completions/Completions APIs report
// prompt_tokens/completion_tokens, while the Responses API (/v1/responses) reports
// the same two semantic values under input_tokens/output_tokens. Both naming pairs
// are therefore aliases for the same concepts:
// prompt_tokens == input_tokens (tokens in the request)
// completion_tokens == output_tokens (tokens generated)
// Only one pair is populated per response depending on the upstream API. Fields are
// pointers so an absent field (nil) is distinguishable from a genuine zero count,
// which lets the prompt/input (and completion/output) fallback select the right alias.
Usage *struct {
PromptTokens *int64 `json:"prompt_tokens"`
CompletionTokens *int64 `json:"completion_tokens"`
TotalTokens *int64 `json:"total_tokens"`
InputTokens *int64 `json:"input_tokens"`
OutputTokens *int64 `json:"output_tokens"`
} `json:"usage"`
Code int `json:"code"`
}
type Server ¶
type Server struct {
// contains filtered or unexported fields
}
func NewServer ¶
func NewServer(redisClient *redis.Client, client kubernetes.Interface, gatewayClient gatewayapi.Interface) *Server
func (*Server) HandleRequestBody ¶
func (s *Server) HandleRequestBody(ctx context.Context, routingCtx *types.RoutingContext, requestID string, req *extProcPb.ProcessingRequest, user utils.User) (*extProcPb.ProcessingResponse, string, bool, int64)
func (*Server) HandleRequestHeaders ¶
func (s *Server) HandleRequestHeaders(ctx context.Context, requestID string, req *extProcPb.ProcessingRequest) (*extProcPb.ProcessingResponse, utils.User, int64, *types.RoutingContext)
func (*Server) HandleResponseBody ¶
func (*Server) HandleResponseHeaders ¶
func (s *Server) HandleResponseHeaders(ctx context.Context, routerCtx *types.RoutingContext, requestID string, model string, req *extProcPb.ProcessingRequest) (*extProcPb.ProcessingResponse, bool, int)
func (*Server) Process ¶
func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error
func (*Server) StartHTTPServer ¶ added in v0.7.0
StartHTTPServer starts the gateway's HTTP server with metrics and API handlers. In local/standalone mode, Envoy routes /v1/models here since there is no metadata service. In standard K8s deployment, Envoy routes /v1/models to the metadata service instead, so the /v1/models handler here is never reached — no conflict.
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
pd
Package pd contains the scoring types used by the PD (prefill-decode) disaggregated-inference router.
|
Package pd contains the scoring types used by the PD (prefill-decode) disaggregated-inference router. |
|
pd/selector
Package selector defines the PodSelector contract for PD-disaggregated routing.
|
Package selector defines the PodSelector contract for PD-disaggregated routing. |
|
vtc
Package vtc implements the Virtual Token Counter routing algorithms focused on fairness and utilization
|
Package vtc implements the Virtual Token Counter routing algorithms focused on fairness and utilization |
|
Package configprofiles parses the model.aibrix.ai/config annotation (or ConfigMap) and supports multiple named profiles selectable at runtime via config-profile header.
|
Package configprofiles parses the model.aibrix.ai/config annotation (or ConfigMap) and supports multiple named profiles selectable at runtime via config-profile header. |
|
Package statesync provides a generic Redis-backed state sync layer for use across multiple gateway (or other) replicas.
|
Package statesync provides a generic Redis-backed state sync layer for use across multiple gateway (or other) replicas. |
Click to show internal directories.
Click to hide internal directories.