Documentation
¶
Index ¶
Constants ¶
const ( NVCFSBSZoneDNSEnv = "NVCF_SBS_ZONE_DNS" NVCFStreamingInterfaceEnv = "NVCF_LLS_INTERFACE" )
const ( FunctionTypeDefault = "DEFAULT" FunctionTypeStreaming = "STREAMING" FunctionTypeLLM = "LLM" // Volume mount paths InferenceDirPath = "/var/inf" ConfigDirPath = "/config/shared" )
const (
LLMWorkerContainerName = "llm-worker"
)
Variables ¶
This section is empty.
Functions ¶
func Translate ¶
func Translate(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error)
Types ¶
type CreationQueueMessage ¶
type CreationQueueMessage struct {
common.CreationQueueMessageMetadata `json:",inline"`
Details Details `json:"functionDetails"`
LaunchArtifacts LaunchArtifacts `json:"launchArtifacts,omitempty"`
LaunchSpecification *LaunchSpecification `json:"launchSpecification,omitempty"`
}
func (CreationQueueMessage) GetCreationQueueMessageMetadata ¶
func (m CreationQueueMessage) GetCreationQueueMessageMetadata() common.CreationQueueMessageMetadata
type Details ¶
type Details struct {
FunctionID string `json:"functionId"`
FunctionVersionID string `json:"functionVersionId"`
FunctionType string `json:"functionType"`
}
+k8s:deepcopy-gen=true
func (*Details) DeepCopy ¶
DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Details.
func (*Details) DeepCopyInto ¶
DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
type LLMModelConfig ¶
type LLMModelConfig struct {
URIs []string `json:"uris"`
Tokenizer string `json:"tokenizer"`
TokenRateLimit string `json:"tokenRateLimit"`
RoutingMethod string `json:"routingMethod"`
}
LLMModelConfig is the LLM function type-specific configuration. +k8s:deepcopy-gen=true
func (*LLMModelConfig) DeepCopy ¶
func (in *LLMModelConfig) DeepCopy() *LLMModelConfig
DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMModelConfig.
func (*LLMModelConfig) DeepCopyInto ¶
func (in *LLMModelConfig) DeepCopyInto(out *LLMModelConfig)
DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
type LaunchArtifact ¶
type LaunchArtifact struct {
Type LaunchArtifactType `json:"artifactType,omitempty"`
Specification string `json:"artifactSpec,omitempty"`
}
type LaunchArtifactType ¶
type LaunchArtifactType string
const ( LaunchArtifactTypePod LaunchArtifactType = "POD_SPEC" LaunchArtifactTypeSecret LaunchArtifactType = "SECRET_SPEC" LaunchArtifactTypeConfigmap LaunchArtifactType = "CONFIGMAP_SPEC" LaunchArtifactTypeInitCacheJob LaunchArtifactType = "INIT_CACHE_JOB_SPEC" LaunchArtifactTypeBlockDevice LaunchArtifactType = "BD_CREATE_SPEC" LaunchArtifactTypeHelmChart LaunchArtifactType = "HELM_CHART" LaunchArtifactTypeHelmCreds LaunchArtifactType = "HELM_CREDS" )
type LaunchArtifacts ¶
type LaunchArtifacts []LaunchArtifact
+k8s:deepcopy-gen=true
func (LaunchArtifacts) DeepCopy ¶
func (in LaunchArtifacts) DeepCopy() LaunchArtifacts
DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LaunchArtifacts.
func (LaunchArtifacts) DeepCopyInto ¶
func (in LaunchArtifacts) DeepCopyInto(out *LaunchArtifacts)
DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
type LaunchSpecification ¶
type LaunchSpecification struct {
// Container function details are baked into the environment.
EnvironmentB64 string `json:"environment"`
ICMSEnvironment string `json:"icmsEnvironment"`
CloudProvider string `json:"cloudProvider"`
GPUName string `json:"gpuName"`
// MaxRequestConcurrency is the per-instance limit from the NVCF deployment GPU specification.
MaxRequestConcurrency int `json:"maxRequestConcurrency,omitempty"`
// Helm chart function components of the launch spec.
*common.HelmChartLaunchSpecification `json:",inline"`
// Cache object configuration metadata of the launch spec.
*common.CacheLaunchSpecification `json:",inline"`
// Telemetry configuration metadata.
Telemetries *common.TelemetriesLaunchSpecification `json:"telemetries,omitempty"`
// Models launch specification metadata.
// This is only used by the translate lib for LLM functions currently.
Models Models `json:"models,omitempty"`
}
+k8s:deepcopy-gen=true
func (*LaunchSpecification) DeepCopy ¶
func (in *LaunchSpecification) DeepCopy() *LaunchSpecification
DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LaunchSpecification.
func (*LaunchSpecification) DeepCopyInto ¶
func (in *LaunchSpecification) DeepCopyInto(out *LaunchSpecification)
DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
type Model ¶
type Model struct {
Name string `json:"name"`
Version string `json:"version"`
URI string `json:"uri"`
LLMModelConfig *LLMModelConfig `json:"llmConfig,omitempty"`
}
Model is a specification for a specific model. +k8s:deepcopy-gen=true
func (*Model) DeepCopy ¶
DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Model.
func (*Model) DeepCopyInto ¶
DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
type Models ¶
type Models []Model
Models is the launch specification for function models. +k8s:deepcopy-gen=true
func (Models) DeepCopy ¶
DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Models.
func (Models) DeepCopyInto ¶
DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (*Models) UnmarshalJSON ¶
type TranslateConfig ¶
type TranslateConfig struct {
common.TranslateConfig `json:",inline"`
PodUseHostNetwork bool `json:"podUseHostNetwork,omitempty"`
PodDNSPolicy v1.DNSPolicy `json:"podDNSPolicy,omitempty"`
// enable this to create side-cars as Deployment instead-of
// barepod for non-disruptive worker upgrades
SidecarAsDeployment bool `json:"sidecarAsDeployment"`
// Control Helm workload secret translation.
// Setting this to true means that the caller should use common.FilterObjectImageRegistryAuths
// to find only relevant secrets to translation.
DisableHelmWorkloadSecretTranslation bool
// Deprecated: LLM request router address must be supplied by environment.
DefaultStargateAddress string `json:"defaultStargateAddress,omitempty"`
// Whether to use QUIC insecure mode for stargate connections
// If not set, "--quick-insecure" is omitted
StargateQUICInsecure bool `json:"stargateQUICInsecure,omitempty"`
}
Source Files