core

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 23, 2025 License: Apache-2.0 Imports: 5 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetAccelerators 

func GetAccelerators() map[string]*Accelerator

func GetCapacities 

func GetCapacities() map[string]int

func GetModels 

func GetModels() map[string]*Model

func GetServers 

func GetServers() map[string]*Server

Types

type Accelerator 

type Accelerator struct {
	// contains filtered or unexported fields
}

An accelerator used in an inference server

  • full or multiple GPU units (cards)

func GetAccelerator 

func GetAccelerator(name string) *Accelerator

func NewAcceleratorFromSpec 

func NewAcceleratorFromSpec(spec *config.AcceleratorSpec) *Accelerator

func (*Accelerator) Calculate 

func (g *Accelerator) Calculate()

Calculate basic parameters

func (*Accelerator) Cost 

func (g *Accelerator) Cost() float32

func (*Accelerator) MemSize 

func (g *Accelerator) MemSize() int

func (*Accelerator) Multiplicity 

func (g *Accelerator) Multiplicity() int

func (*Accelerator) Name 

func (g *Accelerator) Name() string

func (*Accelerator) Power 

func (g *Accelerator) Power(util float32) float32

Evaluate power consumption at a given utilization

func (*Accelerator) Spec 

func (g *Accelerator) Spec() *config.AcceleratorSpec

func (*Accelerator) String 

func (g *Accelerator) String() string

func (*Accelerator) Type 

func (g *Accelerator) Type() string

type Allocation 

type Allocation struct {
	// contains filtered or unexported fields
}

Allocation details of an accelerator to a server

func AllocationFromData 

func AllocationFromData(data *config.AllocationData) *Allocation

func CreateAllocation 

func CreateAllocation(serverName string, gName string) *Allocation

Create an allocation of an accelerator to a server; nil if not feasible

func (*Allocation) Accelerator 

func (a *Allocation) Accelerator() string

func (*Allocation) AllocationData 

func (a *Allocation) AllocationData() *config.AllocationData

func (*Allocation) Clone 

func (a *Allocation) Clone() *Allocation

func (*Allocation) Cost 

func (a *Allocation) Cost() float32

func (*Allocation) MaxArrvRatePerReplica 

func (a *Allocation) MaxArrvRatePerReplica() float32

func (*Allocation) MaxBatchSize 

func (a *Allocation) MaxBatchSize() int

func (*Allocation) MaxRPM 

func (a *Allocation) MaxRPM() float32

func (*Allocation) NumReplicas 

func (a *Allocation) NumReplicas() int

func (*Allocation) ReAllocate 

func (a *Allocation) ReAllocate(serverName string) (*Allocation, string)

func (*Allocation) Saturated 

func (a *Allocation) Saturated(totalRate float32) bool

func (*Allocation) Scale 

func (a *Allocation) Scale(serverName string) (alloc *Allocation, inc int)

func (*Allocation) SetCost 

func (a *Allocation) SetCost(cost float32)

func (*Allocation) SetMaxBatchSize 

func (a *Allocation) SetMaxBatchSize(batchSize int)

func (*Allocation) SetNumReplicas 

func (a *Allocation) SetNumReplicas(n int)

func (*Allocation) SetValue 

func (a *Allocation) SetValue(value float32)

Set the value for this allocation (may depend on cost, performance, ...)

func (*Allocation) String 

func (a *Allocation) String() string

func (*Allocation) TransitionPenalty 

func (a *Allocation) TransitionPenalty(b *Allocation) float32

Calculate penalty for transitioning from this allocation (a) to another allocation (b)

func (*Allocation) Value 

func (a *Allocation) Value() float32

type AllocationByType 

type AllocationByType struct {
	// contains filtered or unexported fields
}

Allocation data about an accelerator type

func (*AllocationByType) String 

func (a *AllocationByType) String() string

type AllocationDiff 

type AllocationDiff struct {
	// contains filtered or unexported fields
}

Orchestration difference between two allocations

func CreateAllocationDiff 

func CreateAllocationDiff(a *Allocation, b *Allocation) *AllocationDiff

func (*AllocationDiff) String 

func (d *AllocationDiff) String() string

type Model 

type Model struct {
	// contains filtered or unexported fields
}

An inference model

func GetModel 

func GetModel(name string) *Model

func NewModel 

func NewModel(name string) *Model

func (*Model) AddPerfDataFromSpec 

func (m *Model) AddPerfDataFromSpec(spec *config.ModelAcceleratorPerfData)

func (*Model) Calculate 

func (m *Model) Calculate(accelerators map[string]*Accelerator)

Calculate basic parameters

func (*Model) Name 

func (m *Model) Name() string

func (*Model) NumInstances 

func (m *Model) NumInstances(acceleratorName string) int

func (*Model) PerfData 

func (m *Model) PerfData(acceleratorName string) *config.ModelAcceleratorPerfData

func (*Model) RemovePerfData 

func (m *Model) RemovePerfData(accName string)

func (*Model) Spec 

func (m *Model) Spec() *config.ModelData

func (*Model) String 

func (m *Model) String() string

type Server 

type Server struct {
	// contains filtered or unexported fields
}

A server for a service class and model

func GetServer 

func GetServer(name string) *Server

func NewServerFromSpec 

func NewServerFromSpec(spec *config.ServerSpec) *Server

func (*Server) AllAllocations 

func (s *Server) AllAllocations() map[string]*Allocation

func (*Server) Allocation 

func (s *Server) Allocation() *Allocation

func (*Server) ApplyDesiredAlloc 

func (s *Server) ApplyDesiredAlloc()

func (*Server) Calculate 

func (s *Server) Calculate(accelerators map[string]*Accelerator)

Calculate allocations for a set of accelerators

func (*Server) CurAllocation 

func (s *Server) CurAllocation() *Allocation

func (*Server) GetCandidateAccelerators 

func (s *Server) GetCandidateAccelerators(accelerators map[string]*Accelerator) map[string]*Accelerator

Create a subset of candidate accelerators for a server from a given set

func (*Server) KeepAccelerator 

func (s *Server) KeepAccelerator() bool

func (*Server) Load 

func (s *Server) Load() *config.ServerLoadSpec

func (*Server) ModelName 

func (s *Server) ModelName() string

func (*Server) Name 

func (s *Server) Name() string

func (*Server) Priority 

func (s *Server) Priority() int

func (*Server) RemoveAllocation 

func (s *Server) RemoveAllocation()

func (*Server) Saturated 

func (s *Server) Saturated() bool

func (*Server) ServiceClassName 

func (s *Server) ServiceClassName() string

func (*Server) SetAllocation 

func (s *Server) SetAllocation(alloc *Allocation)

func (*Server) SetCurAllocation 

func (s *Server) SetCurAllocation(curAllocation *Allocation)

func (*Server) SetLoad 

func (s *Server) SetLoad(load *config.ServerLoadSpec)

func (*Server) Spec 

func (s *Server) Spec() *config.ServerSpec

func (*Server) String 

func (s *Server) String() string

func (*Server) UpdateDesiredAlloc 

func (s *Server) UpdateDesiredAlloc()

type ServiceClass 

type ServiceClass struct {
	// contains filtered or unexported fields
}

A service class

func GetServiceClass 

func GetServiceClass(name string) *ServiceClass

func NewServiceClass 

func NewServiceClass(name string, priority int) *ServiceClass

func NewServiceClassFromSpec 

func NewServiceClassFromSpec(spec *config.ServiceClassSpec) *ServiceClass

func (*ServiceClass) AddModelTarget 

func (c *ServiceClass) AddModelTarget(spec *config.ModelTarget) *Target

add a model target to the service class (replace if already exists)

func (*ServiceClass) ModelTarget 

func (c *ServiceClass) ModelTarget(modelName string) *Target

func (*ServiceClass) Name 

func (c *ServiceClass) Name() string

func (*ServiceClass) Priority 

func (c *ServiceClass) Priority() int

func (*ServiceClass) RemoveModelTarget 

func (c *ServiceClass) RemoveModelTarget(modelName string)

func (*ServiceClass) Spec 

func (c *ServiceClass) Spec() config.ServiceClassSpec

func (*ServiceClass) String 

func (c *ServiceClass) String() string

func (*ServiceClass) UpdateModelTargets 

func (c *ServiceClass) UpdateModelTargets(spec *config.ServiceClassSpec) bool

update model targets from service class specification (replace if already exists)

type System 

type System struct {
	// contains filtered or unexported fields
}

System comprising all accelerators, models, service classes, and servers 

var (
	// a static reference to the singleton system object
	TheSystem *System
)

func NewSystem 

func NewSystem() *System

Create a new system

func (*System) Accelerator 

func (s *System) Accelerator(name string) *Accelerator

Get accelerator object for a given accelerator name; nil if doesn't exist

func (*System) Accelerators 

func (s *System) Accelerators() map[string]*Accelerator

Get all accelerators

func (*System) AddAcceleratorFromSpec 

func (s *System) AddAcceleratorFromSpec(spec config.AcceleratorSpec)

Add an accelerator (replace if already exists)

func (*System) AddModel 

func (s *System) AddModel(name string) *Model

Add a model (replace if already exists)

func (*System) AddServerFromSpec 

func (s *System) AddServerFromSpec(spec config.ServerSpec)

Add a server (replace if already exists)

func (*System) AddServiceClass 

func (s *System) AddServiceClass(name string, priority int)

Add a service class (replace if already exists)

func (*System) AllocateByType 

func (s *System) AllocateByType()

Accumulate allocation data by accelerator type

func (*System) Calculate 

func (s *System) Calculate()

Calculate basic parameters

func (*System) Capacities 

func (s *System) Capacities() map[string]int

Get capacities of accelerator types

func (*System) Capacity 

func (s *System) Capacity(name string) (int, bool)

Get capacity of an accelerator type

func (*System) GenerateSolution 

func (s *System) GenerateSolution() *config.AllocationSolution

generate json allocation solution for all servers in the system

func (*System) Model 

func (s *System) Model(name string) *Model

Get model object for a given model name; nil if doesn't exist

func (*System) Models 

func (s *System) Models() map[string]*Model

Get all models

func (*System) RemoveAccelerator 

func (s *System) RemoveAccelerator(name string) error

Remove an accelerator

func (*System) RemoveCapacity 

func (s *System) RemoveCapacity(name string) bool

Remove capacity of an accelerator type

func (*System) RemoveModel 

func (s *System) RemoveModel(name string) error

Remove a model

func (*System) RemoveServer 

func (s *System) RemoveServer(name string) error

Remove a server

func (*System) RemoveServiceClass 

func (s *System) RemoveServiceClass(name string) error

Remove a service class

func (*System) Server 

func (s *System) Server(name string) *Server

Get server object for a given server name; nil if doesn't exist

func (*System) Servers 

func (s *System) Servers() map[string]*Server

Get all servers

func (*System) ServiceClass 

func (s *System) ServiceClass(name string) *ServiceClass

Get service class object for a given service class name; nil if doesn't exist

func (*System) ServiceClasses 

func (s *System) ServiceClasses() map[string]*ServiceClass

Get all service classes

func (*System) SetAcceleratorsFromSpec 

func (s *System) SetAcceleratorsFromSpec(d *config.AcceleratorData)

Set accelerators from spec

func (*System) SetCapacityFromSpec 

func (s *System) SetCapacityFromSpec(d *config.CapacityData)

Set capacity count from spec

func (*System) SetCountFromSpec 

func (s *System) SetCountFromSpec(spec config.AcceleratorCount)

Set capacity count for an accelerator type

func (*System) SetFromSpec 

func (s *System) SetFromSpec(d *config.SystemSpec) *config.OptimizerSpec

Set system from spec

func (*System) SetModelsFromSpec 

func (s *System) SetModelsFromSpec(d *config.ModelData)

Set models from spec

func (*System) SetServersFromSpec 

func (s *System) SetServersFromSpec(d *config.ServerData)

Set servers from spec

func (*System) SetServiceClassesFromSpec 

func (s *System) SetServiceClassesFromSpec(d *config.ServiceClassData)

Set service classes from spec

func (*System) String 

func (s *System) String() string

type Target 

type Target struct {
	ITL  float32
	TTFT float32
	TPS  float32
}

target SLOs for service class

func (*Target) String 

func (t *Target) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.