Documentation
¶
Overview ¶
Copyright (c) Microsoft Corporation. Licensed under the MIT license.
Index ¶
Constants ¶
View Source
const ( RuntimeNameHuggingfaceTransformers RuntimeName = "transformers" RuntimeNameVLLM RuntimeName = "vllm" ConfigfileNameVLLM = "inference_config.yaml" DefaultMemoryUtilVLLM = 0.9 // Default gpu memory utilization for VLLM runtime UpperMemoryUtilVLLM = 0.95 // PortRayCluster is the default port for communication between the head and worker nodes in a Ray cluster. PortRayCluster = 6379 )
Variables ¶
View Source
var ( // vLLM will do kvcache pre-allocation, // We need to reserve enough memory for other ephemeral operations to avoid OOM. // This is an empirical value. ReservedNonKVCacheMemory = resource.MustParse("1.5Gi") )
Functions ¶
This section is empty.
Types ¶
type HuggingfaceTransformersParam ¶
type HuggingfaceTransformersParam struct {
BaseCommand string // The initial command (e.g., 'accelerate launch') used in the command line.
AccelerateParams map[string]string // Parameters for configuring the accelerate command.
InferenceMainFile string // The main file for inference.
ModelRunParams map[string]string // Parameters for running the model training/inference.
}
func (*HuggingfaceTransformersParam) DeepCopy ¶
func (h *HuggingfaceTransformersParam) DeepCopy() HuggingfaceTransformersParam
type Metadata ¶ added in v0.4.6
type Metadata struct {
// Name is the name of the model, which serves as a unique identifier.
// It is used to register the model information and retrieve it later.
Name string `yaml:"name"`
// ModelType is the type of the model, which indicates the kind of model
// it is. Currently, the only supported types are "text-generation" and
// "llama2-completion" (deprecated).
ModelType string `yaml:"type"`
// Version is the version of the model. It is a URL that points to the
// model's huggingface page, which contains the model's repository ID
// and revision ID, e.g. https://huggingface.co/mistralai/Mistral-7B-v0.3/commit/d8cadc02ac76bd617a919d50b092e59d2d110aff.
Version string `yaml:"version"`
// Runtime is the runtime environment in which the model operates.
// Currently, the only supported runtime is "tfs".
Runtime string `yaml:"runtime"`
// DownloadAtRuntime indicates whether the model should be downloaded
// at runtime. If set to true, the model will be downloaded when the
// model deployment is created, and the container image will always be
// the Kaito base image. If set to false, a container image whose name
// contains the model name will be used, in which the model weights are baked.
// +optional
DownloadAtRuntime bool `yaml:"downloadAtRuntime,omitempty"`
// Tag is the tag of the container image used to run the model.
// If the model uses the Kaito base image, the tag field can be ignored
// +optional
Tag string `yaml:"tag,omitempty"`
}
Metadata defines the metadata for a model.
type Model ¶
type Model interface {
// GetInferenceParameters returns the preset inference parameters for the model.
GetInferenceParameters() *PresetParam
// GetTuningParameters returns the preset tuning parameters for the model.
GetTuningParameters() *PresetParam
// SupportDistributedInference checks if the model supports distributed inference.
// If true, the inference workload will use 'StatefulSet' instead of 'Deployment'
// as the workload type. See https://github.com/kaito-project/kaito/blob/main/docs/proposals/20250325-distributed-inference.md
// for more details.
SupportDistributedInference() bool
// SupportTuning checks if the model supports tuning.
SupportTuning() bool
}
type PresetParam ¶
type PresetParam struct {
Metadata
DiskStorageRequirement string // Disk storage requirements for the model.
ImageAccessMode string // Defines where the Image is Public or Private.
GPUCountRequirement string // Number of GPUs required for the Preset. Used for inference.
TotalGPUMemoryRequirement string // Total GPU memory required for the Preset. Used for inference.
PerGPUMemoryRequirement string // GPU memory required per GPU. Used for inference.
TuningPerGPUMemoryRequirement map[string]int // Min GPU memory per tuning method (batch size 1). Used for tuning.
RuntimeParam
// ReadinessTimeout defines the maximum duration for creating the workload.
// This timeout accommodates the size of the image, ensuring pull completion
// even under slower network conditions or unforeseen delays.
ReadinessTimeout time.Duration
}
PresetParam defines the preset inference parameters for a model.
func (*PresetParam) DeepCopy ¶
func (p *PresetParam) DeepCopy() *PresetParam
func (*PresetParam) GetInferenceCommand ¶
func (p *PresetParam) GetInferenceCommand(rc RuntimeContext) []string
func (*PresetParam) Validate ¶ added in v0.4.5
func (p *PresetParam) Validate(rc RuntimeContext) error
type RuntimeContext ¶ added in v0.4.5
type RuntimeContext struct {
RuntimeName RuntimeName
GPUConfig *sku.GPUConfig
ConfigVolume *corev1.VolumeMount
SKUNumGPUs int
NumNodes int
WorkspaceMetadata metav1.ObjectMeta
DistributedInference bool
RuntimeContextExtraArguments
}
RuntimeContext defines the runtime context for a model.
type RuntimeContextExtraArguments ¶ added in v0.4.5
type RuntimeParam ¶
type RuntimeParam struct {
Transformers HuggingfaceTransformersParam
VLLM VLLMParam
// Disable the tensor parallelism
DisableTensorParallelism bool
}
RuntimeParam defines the llm runtime parameters.
func (*RuntimeParam) DeepCopy ¶
func (rp *RuntimeParam) DeepCopy() RuntimeParam
type VLLMParam ¶
type VLLMParam struct {
RayLeaderBaseCommand string
RayLeaderParams map[string]string
RayWorkerBaseCommand string
RayWorkerParams map[string]string
// BaseCommand is the command used to start the inference server.
BaseCommand string
// The model name used in the openai serving API.
// see https://platform.openai.com/docs/api-reference/chat/create#chat-create-model.
ModelName string
// Parameters for running the model training/inference.
ModelRunParams map[string]string
// Indicates if vllm supports LoRA (Low-Rank Adaptation) for this model.
// doc: https://docs.vllm.ai/en/latest/models/supported_models.html#text-generation-task-generate
DisallowLoRA bool
}
Click to show internal directories.
Click to hide internal directories.