Documentation
¶
Index ¶
- Variables
- func ByLibrary(l []DeviceInfo) [][]DeviceInfo
- func FlashAttentionSupported(l []DeviceInfo) bool
- func GetDevicesEnv(l []DeviceInfo) map[string]string
- func LibraryPaths(l []DeviceInfo) []string
- func SimilarDeviceDescription(a, b string) bool
- func SimilarDeviceMemory(a, b uint64) bool
- type BaseRunner
- type ByFreeMemory
- type DeviceComparison
- type DeviceID
- type DeviceInfo
- func (d DeviceInfo) AddInitValidation(env map[string]string)
- func (a DeviceInfo) Compare(b DeviceInfo) DeviceComparison
- func (d DeviceInfo) Compute() string
- func (d DeviceInfo) Driver() string
- func (d DeviceInfo) MinimumMemory() uint64
- func (d DeviceInfo) NeedsInitValidation() bool
- func (d DeviceInfo) PreferredLibrary(other DeviceInfo) bool
- type FilteredRunnerDiscovery
- type FlashAttentionType
- type RunnerDiscovery
- type SystemInfo
Constants ¶
This section is empty.
Variables ¶
var LibOllamaPath = func() string { exe, err := os.Executable() if err != nil { return "" } if eval, err := filepath.EvalSymlinks(exe); err == nil { exe = eval } cwd, err := os.Getwd() if err != nil { cwd = "" } return findLibOllamaPath(libOllamaPathSearch{ executable: exe, workingDir: cwd, goos: runtime.GOOS, goarch: runtime.GOARCH, }) }()
LibOllamaPath is the root used to find bundled llama.cpp and MLX runtime libraries. GPU-specific libraries live in backend subdirectories such as cuda_v12, rocm_v7_2, vulkan, and mlx_cuda_v13.
Functions ¶
func ByLibrary ¶ added in v0.12.7
func ByLibrary(l []DeviceInfo) [][]DeviceInfo
func FlashAttentionSupported ¶ added in v0.12.7
func FlashAttentionSupported(l []DeviceInfo) bool
FlashAttentionSupported reports whether flash attention can be used across all selected devices.
func GetDevicesEnv ¶ added in v0.23.0
func GetDevicesEnv(l []DeviceInfo) map[string]string
Given the list of GPUs this instantiation is targeted for, figure out the device environment variables and any recorded per-device runner environment overrides.
func LibraryPaths ¶ added in v0.12.7
func LibraryPaths(l []DeviceInfo) []string
func SimilarDeviceDescription ¶ added in v0.30.0
SimilarDeviceDescription reports whether two backend device descriptions are close enough to identify the same physical GPU across different libraries.
func SimilarDeviceMemory ¶ added in v0.30.0
Types ¶
type BaseRunner ¶ added in v0.12.7
type BaseRunner interface {
// GetPort returns the localhost port number the runner is running on
GetPort() int
// HasExited indicates if the runner is no longer running. This can be used during
// bootstrap to detect if a given filtered device is incompatible and triggered an assert
HasExited() bool
}
type ByFreeMemory ¶ added in v0.12.7
type ByFreeMemory []DeviceInfo
Sort by Free Space. iGPUs are reported first, thus Reverse() yields the largest discrete GPU first
func (ByFreeMemory) Len ¶ added in v0.12.7
func (a ByFreeMemory) Len() int
func (ByFreeMemory) Less ¶ added in v0.12.7
func (a ByFreeMemory) Less(i, j int) bool
func (ByFreeMemory) Swap ¶ added in v0.12.7
func (a ByFreeMemory) Swap(i, j int)
type DeviceComparison ¶ added in v0.12.4
type DeviceComparison int
const ( UniqueDevice DeviceComparison = iota SameBackendDevice // The device is the same, and the library/backend is the same DuplicateDevice // The same physical device but different library/backend (overlapping device) )
type DeviceID ¶ added in v0.12.4
type DeviceID struct {
// ID is an identifier for the device for matching with system
// management libraries. The ID is only unique for other devices
// using the same Library.
// This ID represents a "post filtered" view of the enumerated devices
// if the ID is numeric
ID string `json:"id"`
// Library identifies which library is used for the device (e.g. CUDA, ROCm, etc.)
Library string `json:"backend,omitempty"`
}
Minimal unique device identification
type DeviceInfo ¶ added in v0.12.4
type DeviceInfo struct {
DeviceID
// Name is the name of the device as labeled by the backend. It
// may not be persistent across instances of the runner.
Name string `json:"name"`
// Description is the longer user-friendly identification of the device
Description string `json:"description"`
// FilterID is populated with the unfiltered device ID if a numeric ID is used
// so the device can be included.
FilterID string `json:"filter_id,omitempty"`
// Integrated is set true for integrated GPUs, false for Discrete GPUs
Integrated bool `json:"integration,omitempty"`
// PCIID is the bus, device and domain ID of the device for deduplication
// when discovered by multiple backends
PCIID string `json:"pci_id,omitempty"`
// TotalMemory is the total amount of memory the device can use for loading models
TotalMemory uint64 `json:"total_memory"`
// FreeMemory is the amount of memory currently available on the device for loading models
FreeMemory uint64 `json:"free_memory,omitempty"`
// ComputeMajor is the major version of capabilities of the device
// if unsupported by the backend, -1 will be returned
ComputeMajor int
// ComputeMinor is the minor version of capabilities of the device
// if unsupported by the backend, -1 will be returned
ComputeMinor int
// Driver Information
DriverMajor int `json:"driver_major,omitempty"`
DriverMinor int `json:"driver_minor,omitempty"`
// NVIDIADriverMajor is the NVIDIA kernel driver branch. CUDA driver APIs
// expose a separate CUDA compatibility version, so keep this distinct.
NVIDIADriverMajor int `json:"-"`
// GFXTarget is the AMD GPU gfx target string (e.g. "gfx1100") for ROCm
// device validation. Empty on non-AMD devices.
GFXTarget string `json:"gfx_target,omitempty"`
// Where backends were loaded from
LibraryPath []string
// RunnerEnvOverrides stores exceptional per-device runner environment
// overrides discovered during bootstrap. This is internal server state and
// is not serialized.
RunnerEnvOverrides map[string]string `json:"-"`
}
func (DeviceInfo) AddInitValidation ¶ added in v0.12.10
func (d DeviceInfo) AddInitValidation(env map[string]string)
Set the init validation environment variable
func (DeviceInfo) Compare ¶ added in v0.12.4
func (a DeviceInfo) Compare(b DeviceInfo) DeviceComparison
func (DeviceInfo) Compute ¶ added in v0.12.4
func (d DeviceInfo) Compute() string
func (DeviceInfo) Driver ¶ added in v0.12.4
func (d DeviceInfo) Driver() string
func (DeviceInfo) MinimumMemory ¶ added in v0.12.7
func (d DeviceInfo) MinimumMemory() uint64
MinimumMemory reports the amount of memory that should be set aside on the device for overhead (e.g. VRAM consumed by context structures independent of model allocations)
func (DeviceInfo) NeedsInitValidation ¶ added in v0.12.10
func (d DeviceInfo) NeedsInitValidation() bool
NeedsInitValidation returns true if the device in question has the potential to crash at inference time and requires deeper validation before we include it in the supported devices list.
func (DeviceInfo) PreferredLibrary ¶ added in v0.12.10
func (d DeviceInfo) PreferredLibrary(other DeviceInfo) bool
PreferredLibrary returns true if this library is preferred over the other input library Used to filter out Vulkan in favor of CUDA or ROCm
type FilteredRunnerDiscovery ¶ added in v0.12.7
type FilteredRunnerDiscovery interface {
RunnerDiscovery
// GetActiveDeviceIDs returns the filtered set of devices actively in
// use by this runner for running models. If the runner is a bootstrap runner, no devices
// will be active yet so no device IDs are returned.
// This routine will not query the underlying device and will return immediately
GetActiveDeviceIDs() []DeviceID
}
type FlashAttentionType ¶ added in v0.13.4
type FlashAttentionType int32
const ( // Aligned with llama_flash_attn_type FlashAttentionAuto FlashAttentionType = -1 FlashAttentionDisabled FlashAttentionType = 0 FlashAttentionEnabled FlashAttentionType = 1 )
func (FlashAttentionType) LogValue ¶ added in v0.13.4
func (f FlashAttentionType) LogValue() slog.Value
func (FlashAttentionType) String ¶ added in v0.13.4
func (f FlashAttentionType) String() string
type RunnerDiscovery ¶ added in v0.12.7
type RunnerDiscovery interface {
BaseRunner
// GetDeviceInfos will perform a query of the underlying device libraries
// for device identification and free VRAM information
// During bootstrap scenarios, this routine may take seconds to complete
GetDeviceInfos(ctx context.Context) []DeviceInfo
}
type SystemInfo ¶ added in v0.12.7
type SystemInfo struct {
// TotalMemory is the total amount of system memory
TotalMemory uint64 `json:"total_memory,omitempty"`
// FreeMemory is the amount of memory currently available on the system for loading models
FreeMemory uint64 `json:"free_memory,omitempty"`
// FreeSwap is the amount of system swap space reported as available
FreeSwap uint64 `json:"free_swap,omitempty"`
}