Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func RuntimeMemoryCheckEnabled ¶
func RuntimeMemoryCheckEnabled() bool
func SetRuntimeMemoryCheck ¶
func SetRuntimeMemoryCheck(enabled bool)
Types ¶
type MemoryEstimator ¶
type MemoryEstimator interface {
SetDefaultBackend(MemoryEstimatorBackend)
GetRequiredMemoryForModel(context.Context, string, *inference.BackendConfiguration) (inference.RequiredMemory, error)
HaveSufficientMemoryForModel(ctx context.Context, model string, config *inference.BackendConfiguration) (bool, inference.RequiredMemory, inference.RequiredMemory, error)
}
func NewEstimator ¶
func NewEstimator(systemMemoryInfo SystemMemoryInfo) MemoryEstimator
type MemoryEstimatorBackend ¶
type MemoryEstimatorBackend interface {
GetRequiredMemoryForModel(context.Context, string, *inference.BackendConfiguration) (inference.RequiredMemory, error)
}
type SystemMemoryInfo ¶
type SystemMemoryInfo interface {
HaveSufficientMemory(inference.RequiredMemory) (bool, error)
GetTotalMemory() inference.RequiredMemory
}
func NewSystemMemoryInfo ¶
Source Files
Click to show internal directories.
Click to hide internal directories.