gateway

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 18, 2026 License: Apache-2.0 Imports: 54 Imported by: 0

Documentation

Index

Constants

View Source
const (
	HeaderErrorInvalidRouting = "x-error-invalid-routing-strategy"

	// General Error Headers
	HeaderErrorUser                  = "x-error-user"
	HeaderErrorRouting               = "x-error-routing"
	HeaderErrorRequestBodyProcessing = "x-error-request-body-processing"
	HeaderErrorResponseUnmarshal     = "x-error-response-unmarshal"
	HeaderErrorResponseUnknown       = "x-error-response-unknown"

	// Model & Deployment Headers
	HeaderErrorNoModelInRequest = "x-error-no-model-in-request"
	HeaderErrorNoModelBackends  = "x-error-no-model-backends"

	// Streaming Headers
	HeaderErrorStream                    = "x-error-stream"
	HeaderErrorStreaming                 = "x-error-streaming"
	HeaderErrorStreamOptionsIncludeUsage = "x-error-no-stream-options-include-usage"

	// Multipart/Audio Headers
	HeaderErrorMultipartParsing = "x-error-multipart-parsing"

	// Request & Target Headers
	HeaderWentIntoReqHeaders = "x-went-into-req-headers"
	HeaderTargetPodIP        = "target-pod-ip"
	HeaderTargetPod          = "target-pod"
	HeaderRoutingStrategy    = "routing-strategy"
	HeaderRequestID          = "request-id"
	HeaderModel              = "model"
	HeaderExternalFilter     = "external-filter"
	HeaderConfigProfile      = "config-profile"
	// HeaderSessionID is the header used for session affinity routing.
	// NOTE: If you change this value, you MUST also update sessionIDHeader in
	// pkg/plugins/gateway/algorithms/simple_session_affinity.go
	HeaderSessionID   = "x-session-id"
	HeaderTraceParent = "traceparent"

	// RPM & TPM Update Errors
	HeaderUpdateTPM        = "x-update-tpm"
	HeaderUpdateRPM        = "x-update-rpm"
	HeaderErrorRPMExceeded = "x-error-rpm-exceeded"
	HeaderErrorTPMExceeded = "x-error-tpm-exceeded"
	HeaderErrorIncrRPM     = "x-error-incr-rpm"
	HeaderErrorIncrTPM     = "x-error-incr-tpm"

	// Model RPS Errors
	HeaderErrorModelRPSExceeded = "x-error-model-rps-exceeded"
	HeaderErrorIncrModelRPS     = "x-error-incr-model-rps"

	// Rate Limiting defaults
	DefaultRPM           = 100
	DefaultTPMMultiplier = 1000

	// Envs
	EnvRoutingAlgorithm = "ROUTING_ALGORITHM"

	// OpenAI Error Types
	ErrorTypeInvalidRequest = "invalid_request_error"
	ErrorTypeAuthentication = "authentication_error"
	ErrorTypeRateLimit      = "rate_limit_error"
	ErrorTypeApi            = "api_error"
	ErrorTypeOverloaded     = "overloaded_error"

	// OpenAI Error Codes
	ErrorCodeInvalidAPIKey      = "invalid_api_key"
	ErrorCodeModelNotFound      = "model_not_found"
	ErrorCodeRateLimitExceeded  = "rate_limit_exceeded"
	ErrorCodeServiceUnavailable = "service_unavailable"

	// Embedding Constraints
	// https://github.com/openai/openai-go/blob/main/embedding.go#L126
	MaxInputTokensPerModel = 8192
	MaxTotalTokens         = 300000
	MaxArrayDimensions     = 2048

	// Request Paths
	PathChatCompletions     = "/v1/chat/completions"
	PathResponses           = "/v1/responses"
	PathMessages            = "/v1/messages"
	PathCompletions         = "/v1/completions"
	PathEmbeddings          = "/v1/embeddings"
	PathImagesGenerations   = "/v1/images/generations"
	PathVideoGenerations    = "/v1/video/generations"
	PathAudioTranscriptions = "/v1/audio/transcriptions"
	PathAudioTranslations   = "/v1/audio/translations"
	PathRerank              = "/v1/rerank"
	PathClassify            = "/v1/classify"

	// Engine-specific paths (xdit)
	PathXditGenerate      = "/generate"
	PathXditGenerateVideo = "/generatevideo"

	// Engine Types
	EngineXdit = "xdit"
)

Variables

View Source
var (
	ErrorUnknownResponse = errors.New("unknown response")
)
View Source
var (
	POD_NAME = os.Getenv("POD_NAME")
)

Functions

func GetModelTag added in v0.6.0

func GetModelTag(model string) string

func GetTraceID added in v0.7.0

func GetTraceID(traceparent, requestID string) string

Types

type MockCache added in v0.4.0

type MockCache struct {
	mock.Mock
	cache.Cache
}

MockCache implements cache.Cache interface for testing

func (*MockCache) AddRequestCount added in v0.4.0

func (m *MockCache) AddRequestCount(ctx *types.RoutingContext, requestID string, model string) int64

func (*MockCache) AddSubscriber added in v0.4.0

func (m *MockCache) AddSubscriber(subscriber metrics.MetricSubscriber)

func (*MockCache) DoneRequestCount added in v0.4.0

func (m *MockCache) DoneRequestCount(ctx *types.RoutingContext, requestID string, model string, term int64)

func (*MockCache) DoneRequestTrace added in v0.4.0

func (m *MockCache) DoneRequestTrace(ctx *types.RoutingContext, requestID string, model string, inputTokens int64, outputTokens int64, traceTerm int64)

func (*MockCache) GetMetricValueByPod added in v0.4.0

func (m *MockCache) GetMetricValueByPod(namespace string, podName string, metricName string) (metrics.MetricValue, error)

func (*MockCache) GetMetricValueByPodModel added in v0.4.0

func (m *MockCache) GetMetricValueByPodModel(namespace string, podName string, model string, metricName string) (metrics.MetricValue, error)

func (*MockCache) GetPod added in v0.4.0

func (m *MockCache) GetPod(namespace string, podName string) (*v1.Pod, error)

func (*MockCache) HasModel added in v0.4.0

func (m *MockCache) HasModel(model string) bool

func (*MockCache) ListModels added in v0.4.0

func (m *MockCache) ListModels() []string

func (*MockCache) ListModelsByPod added in v0.4.0

func (m *MockCache) ListModelsByPod(namespace string, podName string) ([]string, error)

func (*MockCache) ListPodsByModel added in v0.4.0

func (m *MockCache) ListPodsByModel(model string) (types.PodList, error)

type MockGatewayClassClient added in v0.4.0

type MockGatewayClassClient struct {
	mock.Mock
}

type MockGatewayClient added in v0.4.0

type MockGatewayClient struct {
	mock.Mock
}

MockGatewayClient implements gatewayapi.Clientset interface

func (*MockGatewayClient) Discovery added in v0.4.0

func (*MockGatewayClient) GatewayV1 added in v0.4.0

func (*MockGatewayClient) GatewayV1alpha2 added in v0.4.0

func (*MockGatewayClient) GatewayV1beta1 added in v0.4.0

type MockGatewayV1Client added in v0.4.0

type MockGatewayV1Client struct {
	mock.Mock
}

MockGatewayV1Client implements gatewayapi.Interface

func (*MockGatewayV1Client) GatewayClasses added in v0.4.0

func (*MockGatewayV1Client) Gateways added in v0.4.0

func (*MockGatewayV1Client) HTTPRoutes added in v0.4.0

func (m *MockGatewayV1Client) HTTPRoutes(namespace string) gatewayapiv1.HTTPRouteInterface

func (*MockGatewayV1Client) RESTClient added in v0.4.0

func (m *MockGatewayV1Client) RESTClient() rest.Interface

type MockHTTPRouteClient added in v0.4.0

type MockHTTPRouteClient struct {
	mock.Mock
}

MockHTTPRouteClient implements gatewayapi.HTTPRouteInterface

func (*MockHTTPRouteClient) Create added in v0.4.0

func (*MockHTTPRouteClient) Delete added in v0.4.0

func (m *MockHTTPRouteClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error

func (*MockHTTPRouteClient) DeleteCollection added in v0.4.0

func (m *MockHTTPRouteClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error

func (*MockHTTPRouteClient) Get added in v0.4.0

func (*MockHTTPRouteClient) List added in v0.4.0

func (*MockHTTPRouteClient) Patch added in v0.4.0

func (m *MockHTTPRouteClient) Patch(ctx context.Context, name string, pt k8stype.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *gatewayv1.HTTPRoute, err error)

func (*MockHTTPRouteClient) Update added in v0.4.0

func (*MockHTTPRouteClient) UpdateStatus added in v0.4.0

func (*MockHTTPRouteClient) Watch added in v0.4.0

type OpenAIResponse added in v0.5.0

type OpenAIResponse struct {
	Model string `json:"model"`
	// Usage carries token accounting. The Chat Completions/Completions APIs report
	// prompt_tokens/completion_tokens, while the Responses API (/v1/responses) reports
	// the same two semantic values under input_tokens/output_tokens. Both naming pairs
	// are therefore aliases for the same concepts:
	//   prompt_tokens     == input_tokens   (tokens in the request)
	//   completion_tokens == output_tokens  (tokens generated)
	// Only one pair is populated per response depending on the upstream API. Fields are
	// pointers so an absent field (nil) is distinguishable from a genuine zero count,
	// which lets the prompt/input (and completion/output) fallback select the right alias.
	Usage *struct {
		PromptTokens     *int64 `json:"prompt_tokens"`
		CompletionTokens *int64 `json:"completion_tokens"`
		TotalTokens      *int64 `json:"total_tokens"`
		InputTokens      *int64 `json:"input_tokens"`
		OutputTokens     *int64 `json:"output_tokens"`
	} `json:"usage"`
	Code int `json:"code"`
}

type Server

type Server struct {
	// contains filtered or unexported fields
}

func NewServer

func NewServer(redisClient *redis.Client, client kubernetes.Interface, gatewayClient gatewayapi.Interface) *Server

func (*Server) HandleRequestBody

func (s *Server) HandleRequestBody(ctx context.Context, routingCtx *types.RoutingContext, requestID string, req *extProcPb.ProcessingRequest, user utils.User) (*extProcPb.ProcessingResponse, string, bool, int64)

func (*Server) HandleRequestHeaders

func (*Server) HandleResponseBody

func (s *Server) HandleResponseBody(ctx context.Context, routerCtx *types.RoutingContext, requestID string, req *extProcPb.ProcessingRequest, user utils.User, rpm int64, model string, stream bool, traceTerm int64, hasCompleted bool) (*extProcPb.ProcessingResponse, bool)

func (*Server) HandleResponseHeaders

func (s *Server) HandleResponseHeaders(ctx context.Context, routerCtx *types.RoutingContext, requestID string, model string, req *extProcPb.ProcessingRequest) (*extProcPb.ProcessingResponse, bool, int)

func (*Server) Process

func (*Server) Shutdown added in v0.4.0

func (s *Server) Shutdown()

func (*Server) StartHTTPServer added in v0.7.0

func (s *Server) StartHTTPServer(addr string) error

StartHTTPServer starts the gateway's HTTP server with metrics and API handlers. In local/standalone mode, Envoy routes /v1/models here since there is no metadata service. In standard K8s deployment, Envoy routes /v1/models to the metadata service instead, so the /v1/models handler here is never reached — no conflict.

Directories

Path Synopsis
pd
Package pd contains the scoring types used by the PD (prefill-decode) disaggregated-inference router.
Package pd contains the scoring types used by the PD (prefill-decode) disaggregated-inference router.
pd/selector
Package selector defines the PodSelector contract for PD-disaggregated routing.
Package selector defines the PodSelector contract for PD-disaggregated routing.
vtc
Package vtc implements the Virtual Token Counter routing algorithms focused on fairness and utilization
Package vtc implements the Virtual Token Counter routing algorithms focused on fairness and utilization
Package configprofiles parses the model.aibrix.ai/config annotation (or ConfigMap) and supports multiple named profiles selectable at runtime via config-profile header.
Package configprofiles parses the model.aibrix.ai/config annotation (or ConfigMap) and supports multiple named profiles selectable at runtime via config-profile header.
Package statesync provides a generic Redis-backed state sync layer for use across multiple gateway (or other) replicas.
Package statesync provides a generic Redis-backed state sync layer for use across multiple gateway (or other) replicas.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL