Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

routingalgorithms

package

Go to main page

Versions in this module

v0

v0.5.0-rc.2

Nov 1, 2025

v0.4.1

Aug 19, 2025

Changes in this version

+ const PDRoleSetIdentifier

v0.4.0

Aug 5, 2025

Changes in this version

+ const DefaultFallbackAlgorithm

+ const LLMEngineIdentifier

+ const PDRoleIdentifier

+ const PREBLE_DECODING_LENGTH

+ const PREBLE_EVICTION_LOOP_INTERVAL

+ const PREBLE_SLIDING_WINDOW_PERIOD

+ const PREBLE_TARGET_GPU

+ const PodGroupIndex

+ const RoleReplicaIndex

+ const RouterLeastGpuCache

+ const RouterPD

+ const RouterPrefixCachePreble

+ const RouterSLO

+ const RouterSLOLeastLoad

+ const RouterSLOLeastLoadPulling

+ const RouterSLOPackLoad

+ const RouterUtil

+ const SGLangBootstrapPort

+ const SGLangBootstrapPortIdentifier

+ const SGLangEngine

+ const VLLMEngine

+ var DefaultFallbackRouter types.RouterProviderFunc = RandomRouterProviderFunc

+ var ErrFallbackNotRegistered = errors.New("fallback router not registered")

+ var ErrFallbackNotSupported = errors.New("router not support fallback")

+ var ErrInitTimeout = errors.New("router initialization timeout")

+ var ErrorNoAvailablePod = fmt.Errorf("no pod available")

+ var ModelRouterFactory = NewSLORouter

+ var RandomRouter = &randomRouter

+ var RandomRouterProviderFunc = func(_ *types.RoutingContext) (types.Router, error)

+ func NewLeastGpuCacheRouter() (types.Router, error)

+ func NewLeastLoadPullingRouter(provider cache.CappedLoadProvider) (types.Router, error)

+ func NewLeastLoadRouter(provider cache.LoadProvider) (types.Router, error)

+ func NewLeastUtilRouter() (types.Router, error)

+ func NewPDRouter() (types.Router, error)

+ func NewPackLoadRouter(provider cache.CappedLoadProvider) (types.Router, error)

+ func NewQueueRouter(backend types.Router, queue types.RouterQueue[*types.RoutingContext]) (types.QueueRouter, error)

+ func NewSLORouter(modelName string) (types.QueueRouter, error)

+ func RegisterProvider(algorithm types.RoutingAlgorithm, provider types.RouterProviderFunc)

+ func SetFallback(router types.Router, fallback types.RoutingAlgorithm) error

+ type FallbackRouter struct

+ func (r *FallbackRouter) Route(ctx *types.RoutingContext, pods types.PodList) (string, error)

+ func (r *FallbackRouter) SetFallback(fallback types.RoutingAlgorithm, provider types.RouterProviderFunc)

+ type PrefixCacheMetrics struct

+ type RouterManager struct

+ func NewRouterManager() *RouterManager

+ func (rm *RouterManager) Init()

+ func (rm *RouterManager) Register(algorithm types.RoutingAlgorithm, constructor types.RouterConstructor)

+ func (rm *RouterManager) RegisterProvider(algorithm types.RoutingAlgorithm, provider types.RouterProviderFunc)

+ func (rm *RouterManager) Select(ctx *types.RoutingContext) (types.Router, error)

+ func (rm *RouterManager) SetFallback(router types.Router, fallback types.RoutingAlgorithm) error

+ func (rm *RouterManager) Validate(algorithms string) (types.RoutingAlgorithm, bool)

+ type SLORouter struct

+ func (r *SLORouter) Route(ctx *types.RoutingContext, pods types.PodList) (string, error)

+ type TokenizerPool struct

+ func NewTokenizerPool(config TokenizerPoolConfig, cache cache.Cache) *TokenizerPool

+ func (p *TokenizerPool) Close() error

+ func (p *TokenizerPool) GetTokenizer(model string, pods []*v1.Pod) tokenizer.Tokenizer

+ type TokenizerPoolConfig struct

+ DefaultTokenizer tokenizer.Tokenizer

+ EnableVLLMRemote bool

+ EndpointTemplate string

+ HealthCheckPeriod time.Duration

+ MaxTokenizersPerPool int

+ ModelServiceMap map[string]string

+ Timeout time.Duration

+ TokenizerTTL time.Duration

+ type TokenizerPoolInterface interface

+ Close func() error

+ GetTokenizer func(model string, pods []*v1.Pod) tokenizer.Tokenizer

+ type TokenizerPoolMetrics struct

v0.4.0-rc.4

Aug 2, 2025

v0.4.0-rc.3

Aug 2, 2025

v0.4.0-rc.1

Jul 31, 2025

v0.3.0

May 21, 2025

Changes in this version

+ const RouterNotSet

+ var RouterLeastBusyTime types.RoutingAlgorithm = "least-busy-time"

+ var RouterLeastKvCache types.RoutingAlgorithm = "least-kv-cache"

+ var RouterLeastLatency types.RoutingAlgorithm = "least-latency"

+ var RouterLeastRequest types.RoutingAlgorithm = "least-request"

+ var RouterPrefixCache types.RoutingAlgorithm = "prefix-cache"

+ var RouterPrefixCacheAndLoad types.RoutingAlgorithm = "prefix-cache-preble"

+ var RouterRandom types.RoutingAlgorithm = "random"

+ var RouterThroughput types.RoutingAlgorithm = "throughput"

+ func Init()

+ func NewLeastBusyTimeRouter() (types.Router, error)

+ func NewLeastExpectedLatencyRouter() (types.Router, error)

+ func NewLeastKvCacheRouter() (types.Router, error)

+ func NewLeastRequestRouter() (types.Router, error)

+ func NewPrefixCacheAndLoadRouter() (types.Router, error)

+ func NewPrefixCacheRouter() (types.Router, error)

+ func NewRandomRouter() (types.Router, error)

+ func NewThroughputRouter() (types.Router, error)

+ func Register(algorithm types.RoutingAlgorithm, provider types.RouterProviderFunc)

+ func RegisterDelayed(algorithm types.RoutingAlgorithm, ...)

+ func RegisterDelayedConstructor(algorithm types.RoutingAlgorithm, routerConstructor types.RouterConstructor)

+ func Select(algorithms types.RoutingAlgorithm) types.RouterProviderFunc

+ func SelectRandomPodAsFallback(ctx *types.RoutingContext, pods []*v1.Pod, randomFunc func(int) int) (*v1.Pod, error)

+ func Validate(algorithms string) (types.RoutingAlgorithm, bool)

+ type PrefillTimeParams struct

+ InputIDLens []int

+ NumBatchedTokens int

+ NumRequests int

+ NumUniqueKV int

+ SeqLens []int

+ TotalContext int

+ type SlidingWindowHistogram struct

v0.3.0-rc.2

May 21, 2025

v0.3.0-rc.1

May 13, 2025