gpu

package
v0.9.310 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 9, 2026 License: AGPL-3.0 Imports: 45 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CONTROLLER_PROCESS_NAME                = "cedana-gpu-controller"
	CONTROLLER_ADDRESS_FORMATTER           = "unix://%s/cedana-gpu-controller-%s.sock"
	CONTROLLER_SOCKET_FORMATTER            = "%s/cedana-gpu-controller-%s.sock"
	CONTROLLER_SOCKET_PATTERN              = "cedana-gpu-controller-(.*).sock"
	CONTROLLER_SHM_FILE_FORMATTER          = "/dev/shm/cedana-gpu.%s"
	CONTROLLER_SHM_FILE_PATTERN            = "/dev/shm/cedana-gpu.(.*)"
	CONTROLLER_HOSTMEM_FILE_FORMATTER      = "/dev/shm/cedana-gpu.%s.misc/hostmem-%d"
	CONTROLLER_HOSTMEM_FILE_PATTERN        = "/dev/shm/cedana-gpu.(.*).misc/hostmem-(\\d+)"
	CONTROLLER_MISC_DIR_FORMATTER          = "/dev/shm/cedana-gpu.%s.misc"
	CONTROLLER_BOOKING_LOCK_FILE_FORMATTER = "/dev/shm/cedana-gpu.%s.booking"
	CONTROLLER_TERMINATE_SIGNAL            = syscall.SIGTERM
	CONTROLLER_RESTORE_NEW_PID_SIGNAL      = syscall.SIGUSR1     // Signal to the restored process to notify it has a new PID
	CONTROLLER_CHECK_SHM_SIZE              = 32 * utils.MEBIBYTE // Small size to run controller health check

	LOG_DIR_FORMATTER              = "%s/cedana-gpu.%s"
	LOG_DIR_PATTERN                = "(.*)/cedana-gpu.(.*)"
	INTERCEPTOR_LOG_FILE_PATTERN   = LOG_DIR_PATTERN + "/cedana-so-(\\d+).log"
	INTERCEPTOR_LOG_FILE_FORMATTER = LOG_DIR_FORMATTER + "/cedana-so-%d.log"
	TRACER_LOG_FILE_PATTERN        = LOG_DIR_PATTERN + "/cedana-tracer-(\\d+).log"
	TRACER_LOG_FILE_FORMATTER      = LOG_DIR_FORMATTER + "/cedana-tracer-%d.log"

	FREEZE_TIMEOUT    = 1 * time.Minute
	UNFREEZE_TIMEOUT  = 1 * time.Minute
	DUMP_TIMEOUT      = 10 * time.Minute
	RESTORE_TIMEOUT   = 10 * time.Minute
	HEALTH_TIMEOUT    = 30 * time.Second
	INFO_TIMEOUT      = 30 * time.Second
	TERMINATE_TIMEOUT = 10 * time.Second
	MAX_SYNC_FAILURES = 2
)
View Source 
const (
	CONTROLLER_STALE controllerStatus = iota
	CONTROLLER_FREE
	CONTROLLER_BUSY
)
View Source 
const (
	SYNC_INTERVAL         = 10 * time.Second
	SYNC_SHUTDOWN_TIMEOUT = 45 * time.Second
)

Variables

View Source 
var NVIDIA_MOUNTS_PATTERN = regexp.MustCompile(
	`^(` +
		`/nvidia|` +
		`/dev/nvidia\d+|` +
		`/driver/nvidia/gpus|` +
		`/run/nvidia|` +
		`/usr/bin/nv|` +
		`/usr/lib/firmware/nv|` +
		`/usr/lib(64)?/(x86_64-linux-gnu/|aarch64-linux-gnu/)?(libcuda|libnv)|` +
		`.*nvidia.*|` +
		`/etc/vulkan` +
		`)`,
)

NOTE: Add any other known NVIDIA GPU mount paths here.

Functions

func AddExternalMountsForDump added in v0.9.292 

func AddExternalMountsForDump(next types.Dump) types.Dump

Adapter that tells CRIU about the external GPU mounts.

func AddMountsForRestore added in v0.9.285 

func AddMountsForRestore(next types.Restore) types.Restore

Adapter that tells CRIU about the external GPU mounts.

func Attach 

func Attach(gpus Manager) types.Adapter[types.Run]

Adapter that adds GPU support to the request.

func Dump 

func Dump(gpus Manager) types.Adapter[types.Dump]

Adapter that adds GPU dump to the request.

func EnsureLogDir added in v0.9.268 

func EnsureLogDir(id string, uid, gid uint32) (string, error)

func Freeze added in v0.9.263 

func Freeze(gpus Manager) types.Adapter[types.Freeze]

func InheritFilesForRestore 

func InheritFilesForRestore(next types.Restore) types.Restore

func Interception 

func Interception(next types.Run) types.Run

Adapter that adds GPU interception to the request based on the job type. Each plugin must implement its own support for GPU interception.

func InterceptionRestore added in v0.9.268 

func InterceptionRestore(next types.Restore) types.Restore

Adapter that restore GPU interception to the request based on the job type. Each plugin must implement its own support for restoring GPU interception.

func ProcessInterception 

func ProcessInterception(next types.Run) types.Run

Adapter that adds GPU interception to a process

func ProcessTracing added in v0.9.268 

func ProcessTracing(next types.Run) types.Run

Adapter that adds GPU tracing to a process

func Restore 

func Restore(gpus Manager) types.Adapter[types.Restore]

Adapter that restores GPU support to the request.

func Tracing added in v0.9.268 

func Tracing(next types.Run) types.Run

Adapter that adds GPU tracing to the request based on the job type. Each plugin must implement its own support for GPU tracing.

func TracingRestore added in v0.9.268 

func TracingRestore(next types.Restore) types.Restore

Adapter that restores GPU tracing to the request based on the job type. Each plugin must implement its own support for restoring GPU tracing.

func Unfreeze added in v0.9.263 

func Unfreeze(gpus Manager) types.Adapter[types.Freeze]

Types

type Manager 

type Manager interface {
	// Attach attaches a GPU controller to a process with the given PID.
	// Takes in a channel for the PID, allowing this to be called before the process is started,
	// so that the PID can be passed in later. Returns a unique ID for the GPU controller.
	Attach(ctx context.Context, pid <-chan uint32) (string, error)

	// IsAttached returns true if GPU is attached to a process with the given PID.
	IsAttached(pid uint32) bool

	// Detach detaches the GPU controller from a process with the given and PID.
	Detach(ctx context.Context, pid uint32) error

	// Returns server-compatible health checks.
	Checks() types.Checks

	// GetID returns the ID of the GPU controller for a given PID.
	GetID(pid uint32) string

	// Freeze the GPU for a given PID
	Freeze(ctx context.Context, pid uint32) error

	// Unfreeze the GPU for a given PID
	Unfreeze(ctx context.Context, pid uint32) error

	// CRIUCallback returns the CRIU notify callback for GPU checkpoint/restore.
	CRIUCallback(id string) *criu.NotifyCallback

	// Sync is used to force the GPU manager to sync its state with the current system state.
	Sync(ctx context.Context) error
}

type ManagerMissing 

type ManagerMissing struct{}

Embed this into unimplmented implmentations

func (ManagerMissing) Attach 

func (ManagerMissing) Attach(ctx context.Context, pid <-chan uint32) (string, error)

func (ManagerMissing) CRIUCallback 

func (ManagerMissing) CRIUCallback(id string) *criu.NotifyCallback

func (ManagerMissing) Checks 

func (ManagerMissing) Checks() types.Checks

func (ManagerMissing) Detach 

func (ManagerMissing) Detach(ctx context.Context, pid uint32) error

func (ManagerMissing) Freeze added in v0.9.263 

func (ManagerMissing) Freeze(ctx context.Context, pid uint32) error

func (ManagerMissing) GetID 

func (ManagerMissing) GetID(pid uint32) string

func (ManagerMissing) IsAttached 

func (ManagerMissing) IsAttached(pid uint32) bool

func (ManagerMissing) Sync 

func (ManagerMissing) Sync(ctx context.Context) error

func (ManagerMissing) Unfreeze added in v0.9.263 

func (ManagerMissing) Unfreeze(ctx context.Context, pid uint32) error

type ManagerPool 

type ManagerPool struct {
	*ManagerSimple // Embed simple manager implements most of what we need
	// contains filtered or unexported fields
}

Implements a GPU manager pool that is capable of maintaining a pool of GPU controllers

func NewPoolManager 

func NewPoolManager(lifetime context.Context, serverWg *sync.WaitGroup, poolSize int, plugins plugins.Manager) (*ManagerPool, error)

func (*ManagerPool) Sync 

func (m *ManagerPool) Sync(ctx context.Context) error

type ManagerSimple 

type ManagerSimple struct {
	// contains filtered or unexported fields
}

Implements a simple GPU manager that spawns GPU controllers on-demand

func NewSimpleManager 

func NewSimpleManager(ctx context.Context, serverWg *sync.WaitGroup, plugins plugins.Manager) (*ManagerSimple, error)

func (*ManagerSimple) Attach 

func (m *ManagerSimple) Attach(ctx context.Context, pid <-chan uint32) (id string, err error)

func (*ManagerSimple) CRIUCallback 

func (m *ManagerSimple) CRIUCallback(id string) *criu.NotifyCallback

func (*ManagerSimple) Checks 

func (m *ManagerSimple) Checks() types.Checks

func (*ManagerSimple) Detach 

func (m *ManagerSimple) Detach(ctx context.Context, pid uint32) error

func (*ManagerSimple) Freeze added in v0.9.263 

func (m *ManagerSimple) Freeze(ctx context.Context, pid uint32) error

func (*ManagerSimple) GetID 

func (m *ManagerSimple) GetID(pid uint32) string

func (*ManagerSimple) IsAttached 

func (m *ManagerSimple) IsAttached(pid uint32) bool

func (*ManagerSimple) Sync 

func (m *ManagerSimple) Sync(ctx context.Context) error

func (*ManagerSimple) Unfreeze added in v0.9.263 

func (m *ManagerSimple) Unfreeze(ctx context.Context, pid uint32) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.