model

package
v0.5.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 3, 2025 License: MIT Imports: 12 Imported by: 0

Documentation

Overview

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Index

Constants

View Source
const (
	RuntimeNameHuggingfaceTransformers RuntimeName = "transformers"
	RuntimeNameVLLM                    RuntimeName = "vllm"

	ConfigfileNameVLLM    = "inference_config.yaml"
	DefaultMemoryUtilVLLM = 0.9 // Default gpu memory utilization for VLLM runtime
	UpperMemoryUtilVLLM   = 0.95

	// PortRayCluster is the default port for communication between the head and worker nodes in a Ray cluster.
	PortRayCluster = 6379
)

Variables

View Source
var (
	// vLLM will do kvcache pre-allocation,
	// We need to reserve enough memory for other ephemeral operations to avoid OOM.
	// This is an empirical value.
	ReservedNonKVCacheMemory = resource.MustParse("1.5Gi")
)

Functions

This section is empty.

Types

type HuggingfaceTransformersParam

type HuggingfaceTransformersParam struct {
	BaseCommand       string            // The initial command (e.g., 'accelerate launch') used in the command line.
	AccelerateParams  map[string]string // Parameters for configuring the accelerate command.
	InferenceMainFile string            // The main file for inference.
	ModelRunParams    map[string]string // Parameters for running the model training/inference.
}

func (*HuggingfaceTransformersParam) DeepCopy

type Metadata added in v0.4.6

type Metadata struct {
	// Name is the name of the model, which serves as a unique identifier.
	// It is used to register the model information and retrieve it later.
	Name string `yaml:"name"`

	// ModelType is the type of the model, which indicates the kind of model
	// it is. Currently, the only supported types are "text-generation" and
	// "llama2-completion" (deprecated).
	ModelType string `yaml:"type"`

	// Version is the version of the model. It is a URL that points to the
	// model's huggingface page, which contains the model's repository ID
	// and revision ID, e.g. https://huggingface.co/mistralai/Mistral-7B-v0.3/commit/d8cadc02ac76bd617a919d50b092e59d2d110aff.
	Version string `yaml:"version"`

	// Runtime is the runtime environment in which the model operates.
	// Currently, the only supported runtime is "tfs".
	Runtime string `yaml:"runtime"`

	// DownloadAtRuntime indicates whether the model should be downloaded
	// at runtime. If set to true, the model will be downloaded when the
	// model deployment is created, and the container image will always be
	// the Kaito base image. If set to false, a container image whose name
	// contains the model name will be used, in which the model weights are baked.
	// +optional
	DownloadAtRuntime bool `yaml:"downloadAtRuntime,omitempty"`

	// Tag is the tag of the container image used to run the model.
	// If the model uses the Kaito base image, the tag field can be ignored
	// +optional
	Tag string `yaml:"tag,omitempty"`
}

Metadata defines the metadata for a model.

func (*Metadata) Validate added in v0.4.6

func (m *Metadata) Validate() error

Validate checks if the Metadata is valid.

type Model

type Model interface {
	// GetInferenceParameters returns the preset inference parameters for the model.
	GetInferenceParameters() *PresetParam

	// GetTuningParameters returns the preset tuning parameters for the model.
	GetTuningParameters() *PresetParam

	// SupportDistributedInference checks if the model supports distributed inference.
	// If true, the inference workload will use 'StatefulSet' instead of 'Deployment'
	// as the workload type. See https://github.com/kaito-project/kaito/blob/main/docs/proposals/20250325-distributed-inference.md
	// for more details.
	SupportDistributedInference() bool

	// SupportTuning checks if the model supports tuning.
	SupportTuning() bool
}

type PresetParam

type PresetParam struct {
	Metadata

	DiskStorageRequirement string // Disk storage requirements for the model.

	ImageAccessMode string // Defines where the Image is Public or Private.

	GPUCountRequirement           string         // Number of GPUs required for the Preset. Used for inference.
	TotalGPUMemoryRequirement     string         // Total GPU memory required for the Preset. Used for inference.
	PerGPUMemoryRequirement       string         // GPU memory required per GPU. Used for inference.
	TuningPerGPUMemoryRequirement map[string]int // Min GPU memory per tuning method (batch size 1). Used for tuning.

	RuntimeParam

	// ReadinessTimeout defines the maximum duration for creating the workload.
	// This timeout accommodates the size of the image, ensuring pull completion
	// even under slower network conditions or unforeseen delays.
	ReadinessTimeout time.Duration
}

PresetParam defines the preset inference parameters for a model.

func (*PresetParam) DeepCopy

func (p *PresetParam) DeepCopy() *PresetParam

func (*PresetParam) GetInferenceCommand

func (p *PresetParam) GetInferenceCommand(rc RuntimeContext) []string

func (*PresetParam) Validate added in v0.4.5

func (p *PresetParam) Validate(rc RuntimeContext) error

type RuntimeContext added in v0.4.5

type RuntimeContext struct {
	RuntimeName          RuntimeName
	GPUConfig            *sku.GPUConfig
	ConfigVolume         *corev1.VolumeMount
	SKUNumGPUs           int
	NumNodes             int
	WorkspaceMetadata    metav1.ObjectMeta
	DistributedInference bool
	RuntimeContextExtraArguments
}

RuntimeContext defines the runtime context for a model.

type RuntimeContextExtraArguments added in v0.4.5

type RuntimeContextExtraArguments struct {
	AdaptersEnabled        bool
	AdapterStrengthEnabled bool
}

type RuntimeName

type RuntimeName string

RuntimeName is LLM runtime name.

type RuntimeParam

type RuntimeParam struct {
	Transformers HuggingfaceTransformersParam
	VLLM         VLLMParam
	// Disable the tensor parallelism
	DisableTensorParallelism bool
}

RuntimeParam defines the llm runtime parameters.

func (*RuntimeParam) DeepCopy

func (rp *RuntimeParam) DeepCopy() RuntimeParam

type VLLMParam

type VLLMParam struct {
	RayLeaderBaseCommand string
	RayLeaderParams      map[string]string
	RayWorkerBaseCommand string
	RayWorkerParams      map[string]string
	// BaseCommand is the command used to start the inference server.
	BaseCommand string
	// The model name used in the openai serving API.
	// see https://platform.openai.com/docs/api-reference/chat/create#chat-create-model.
	ModelName string
	// Parameters for running the model training/inference.
	ModelRunParams map[string]string
	// Indicates if vllm supports LoRA (Low-Rank Adaptation) for this model.
	// doc: https://docs.vllm.ai/en/latest/models/supported_models.html#text-generation-task-generate
	DisallowLoRA bool
}

func (*VLLMParam) DeepCopy

func (v *VLLMParam) DeepCopy() VLLMParam

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL