ml

package
v0.34.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 23, 2026 License: MIT Imports: 9 Imported by: 19

Documentation

Index

Constants

This section is empty.

Variables

View Source
var LibOllamaPath = func() string {
	exe, err := os.Executable()
	if err != nil {
		return ""
	}
	if eval, err := filepath.EvalSymlinks(exe); err == nil {
		exe = eval
	}

	cwd, err := os.Getwd()
	if err != nil {
		cwd = ""
	}

	return findLibOllamaPath(libOllamaPathSearch{
		executable: exe,
		workingDir: cwd,
		goos:       runtime.GOOS,
		goarch:     runtime.GOARCH,
	})
}()

LibOllamaPath is the root used to find bundled llama.cpp and MLX runtime libraries. GPU-specific libraries live in backend subdirectories such as cuda_v12, rocm_v7_2, vulkan, and mlx_cuda_v13.

Functions

func ByLibrary added in v0.12.7

func ByLibrary(l []DeviceInfo) [][]DeviceInfo

func FlashAttentionSupported added in v0.12.7

func FlashAttentionSupported(l []DeviceInfo) bool

FlashAttentionSupported reports whether flash attention can be used across all selected devices.

func GetDevicesEnv added in v0.23.0

func GetDevicesEnv(l []DeviceInfo) map[string]string

Given the list of GPUs this instantiation is targeted for, figure out the device environment variables and any recorded per-device runner environment overrides.

func LibraryPaths added in v0.12.7

func LibraryPaths(l []DeviceInfo) []string

func SimilarDeviceDescription added in v0.30.0

func SimilarDeviceDescription(a, b string) bool

SimilarDeviceDescription reports whether two backend device descriptions are close enough to identify the same physical GPU across different libraries.

func SimilarDeviceMemory added in v0.30.0

func SimilarDeviceMemory(a, b uint64) bool

Types

type BaseRunner added in v0.12.7

type BaseRunner interface {
	// GetPort returns the localhost port number the runner is running on
	GetPort() int

	// HasExited indicates if the runner is no longer running.  This can be used during
	// bootstrap to detect if a given filtered device is incompatible and triggered an assert
	HasExited() bool
}

type ByFreeMemory added in v0.12.7

type ByFreeMemory []DeviceInfo

Sort by Free Space. iGPUs are reported first, thus Reverse() yields the largest discrete GPU first

func (ByFreeMemory) Len added in v0.12.7

func (a ByFreeMemory) Len() int

func (ByFreeMemory) Less added in v0.12.7

func (a ByFreeMemory) Less(i, j int) bool

func (ByFreeMemory) Swap added in v0.12.7

func (a ByFreeMemory) Swap(i, j int)

type DeviceComparison added in v0.12.4

type DeviceComparison int
const (
	UniqueDevice      DeviceComparison = iota
	SameBackendDevice                  // The device is the same, and the library/backend is the same
	DuplicateDevice                    // The same physical device but different library/backend (overlapping device)
)

type DeviceID added in v0.12.4

type DeviceID struct {
	// ID is an identifier for the device for matching with system
	// management libraries.  The ID is only unique for other devices
	// using the same Library.
	// This ID represents a "post filtered" view of the enumerated devices
	// if the ID is numeric
	ID string `json:"id"`

	// Library identifies which library is used for the device (e.g. CUDA, ROCm, etc.)
	Library string `json:"backend,omitempty"`
}

Minimal unique device identification

type DeviceInfo added in v0.12.4

type DeviceInfo struct {
	DeviceID

	// Name is the name of the device as labeled by the backend. It
	// may not be persistent across instances of the runner.
	Name string `json:"name"`

	// Description is the longer user-friendly identification of the device
	Description string `json:"description"`

	// FilterID is populated with the unfiltered device ID if a numeric ID is used
	// so the device can be included.
	FilterID string `json:"filter_id,omitempty"`

	// Integrated is set true for integrated GPUs, false for Discrete GPUs
	Integrated bool `json:"integration,omitempty"`

	// PCIID is the bus, device and domain ID of the device for deduplication
	// when discovered by multiple backends
	PCIID string `json:"pci_id,omitempty"`

	// TotalMemory is the total amount of memory the device can use for loading models
	TotalMemory uint64 `json:"total_memory"`

	// FreeMemory is the amount of memory currently available on the device for loading models
	FreeMemory uint64 `json:"free_memory,omitempty"`

	// ComputeMajor is the major version of capabilities of the device
	// if unsupported by the backend, -1 will be returned
	ComputeMajor int

	// ComputeMinor is the minor version of capabilities of the device
	// if unsupported by the backend, -1 will be returned
	ComputeMinor int

	// Driver Information
	DriverMajor int `json:"driver_major,omitempty"`
	DriverMinor int `json:"driver_minor,omitempty"`

	// NVIDIADriverMajor is the NVIDIA kernel driver branch. CUDA driver APIs
	// expose a separate CUDA compatibility version, so keep this distinct.
	NVIDIADriverMajor int `json:"-"`

	// GFXTarget is the AMD GPU gfx target string (e.g. "gfx1100") for ROCm
	// device validation. Empty on non-AMD devices.
	GFXTarget string `json:"gfx_target,omitempty"`

	// Where backends were loaded from
	LibraryPath []string

	// RunnerEnvOverrides stores exceptional per-device runner environment
	// overrides discovered during bootstrap. This is internal server state and
	// is not serialized.
	RunnerEnvOverrides map[string]string `json:"-"`
}

func (DeviceInfo) AddInitValidation added in v0.12.10

func (d DeviceInfo) AddInitValidation(env map[string]string)

Set the init validation environment variable

func (DeviceInfo) Compare added in v0.12.4

func (a DeviceInfo) Compare(b DeviceInfo) DeviceComparison

func (DeviceInfo) Compute added in v0.12.4

func (d DeviceInfo) Compute() string

func (DeviceInfo) Driver added in v0.12.4

func (d DeviceInfo) Driver() string

func (DeviceInfo) MinimumMemory added in v0.12.7

func (d DeviceInfo) MinimumMemory() uint64

MinimumMemory reports the amount of memory that should be set aside on the device for overhead (e.g. VRAM consumed by context structures independent of model allocations)

func (DeviceInfo) NeedsInitValidation added in v0.12.10

func (d DeviceInfo) NeedsInitValidation() bool

NeedsInitValidation returns true if the device in question has the potential to crash at inference time and requires deeper validation before we include it in the supported devices list.

func (DeviceInfo) PreferredLibrary added in v0.12.10

func (d DeviceInfo) PreferredLibrary(other DeviceInfo) bool

PreferredLibrary returns true if this library is preferred over the other input library Used to filter out Vulkan in favor of CUDA or ROCm

type FilteredRunnerDiscovery added in v0.12.7

type FilteredRunnerDiscovery interface {
	RunnerDiscovery

	// GetActiveDeviceIDs returns the filtered set of devices actively in
	// use by this runner for running models.  If the runner is a bootstrap runner, no devices
	// will be active yet so no device IDs are returned.
	// This routine will not query the underlying device and will return immediately
	GetActiveDeviceIDs() []DeviceID
}

type FlashAttentionType added in v0.13.4

type FlashAttentionType int32
const (
	// Aligned with llama_flash_attn_type
	FlashAttentionAuto     FlashAttentionType = -1
	FlashAttentionDisabled FlashAttentionType = 0
	FlashAttentionEnabled  FlashAttentionType = 1
)

func (FlashAttentionType) LogValue added in v0.13.4

func (f FlashAttentionType) LogValue() slog.Value

func (FlashAttentionType) String added in v0.13.4

func (f FlashAttentionType) String() string

type RunnerDiscovery added in v0.12.7

type RunnerDiscovery interface {
	BaseRunner

	// GetDeviceInfos will perform a query of the underlying device libraries
	// for device identification and free VRAM information
	// During bootstrap scenarios, this routine may take seconds to complete
	GetDeviceInfos(ctx context.Context) []DeviceInfo
}

type SystemInfo added in v0.12.7

type SystemInfo struct {
	// TotalMemory is the total amount of system memory
	TotalMemory uint64 `json:"total_memory,omitempty"`

	// FreeMemory is the amount of memory currently available on the system for loading models
	FreeMemory uint64 `json:"free_memory,omitempty"`

	// FreeSwap is the amount of system swap space reported as available
	FreeSwap uint64 `json:"free_swap,omitempty"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL