runtime

package
v0.1.9-alpha Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 1, 2026 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source
const (
	VendorNvidia  = "nvidia"
	VendorAMD     = "amd"
	VendorUnknown = "unknown"
)

GPU vendor identifiers used to pick a vendor-specific runtime build. These mirror hardware.Vendor* but are duplicated here so the runtime package does not depend on hardware detection.

View Source
const (
	BackendCUDA   = "cuda"
	BackendVulkan = "vulkan"
)

llama.cpp compute backends. Manifest artifact keys are named by backend, not by vendor, because the backend is what actually differs between builds and several vendors can share one (Vulkan runs on AMD and Intel alike).

View Source
const DefaultRuntimePublicKeyBase64 = "/Bgu4fQntQGB6q3j18Z+du1L1/yOzU2/wNmSAQWVCMo="

Variables

This section is empty.

Functions

func ArtifactKeys

func ArtifactKeys(platformKey, vendor string) []string

ArtifactKeys lists the manifest keys to try, most specific first. A backend-specific build is preferred when the manifest carries one; the bare platform key stays as the fallback so nodes that predate backend keys, and manifests that never gained them, keep working unchanged.

func BackendForVendor

func BackendForVendor(vendor string) string

BackendForVendor maps a detected GPU vendor onto the runtime build it needs.

An undetected vendor resolves to Vulkan rather than to the bare platform artifact, because the failure modes are not symmetric: the bare Windows artifact is the CUDA build, which cannot run at all on a Radeon, while the Vulkan build runs on NVIDIA too, only slower. When detection fails the safe guess is the one that degrades instead of breaking.

func DecodePublicKey

func DecodePublicKey(encoded string) (ed25519.PublicKey, error)

func DefaultRuntimePublicKey

func DefaultRuntimePublicKey() (ed25519.PublicKey, error)

Types

type ApprovedArgs

type ApprovedArgs struct {
	ContextSize int
	BatchSize   int
	GPULayers   string
	Device      string
	Parallel    int
	Host        string
	Port        int
}

type ChatMessage

type ChatMessage struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type CompletionRequest

type CompletionRequest struct {
	Model       string        `json:"model"`
	Messages    []ChatMessage `json:"messages"`
	Temperature float64       `json:"temperature"`
	MaxTokens   int           `json:"max_tokens"`
	Stream      bool          `json:"stream"`
}

type CompletionResponse

type CompletionResponse struct {
	Choices []struct {
		Message      ChatMessage `json:"message"`
		FinishReason string      `json:"finish_reason"`
	} `json:"choices"`
	Usage CompletionUsage `json:"usage"`
}

func ChatCompletion

func ChatCompletion(ctx context.Context, baseURL string, request CompletionRequest) (CompletionResponse, error)

type CompletionUsage

type CompletionUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

type InstalledRuntime

type InstalledRuntime struct {
	Engine         string `json:"engine"`
	Version        string `json:"version"`
	BuildID        string `json:"build_id"`
	PlatformKey    string `json:"platform_key"`
	Path           string `json:"path"`
	ExecutablePath string `json:"executable_path"`
	SHA256         string `json:"sha256"`
	InstalledAt    string `json:"installed_at"`
}

type LaunchConfig

type LaunchConfig struct {
	ExecutablePath  string
	ModelPath       string
	WorkDir         string
	LogDir          string
	Args            ApprovedArgs
	HealthInterval  time.Duration
	StartupTimeout  time.Duration
	ShutdownTimeout time.Duration
	MaxLogBytes     int64
	Redactions      []string
	Env             []string
}

type Manager

type Manager struct {
	Root        string
	PublicKey   ed25519.PublicKey
	PlatformKey string
	// GPUVendor picks a vendor-specific runtime build when the manifest has
	// one. Empty or "unknown" falls back to the bare platform artifact.
	GPUVendor  string
	HTTPClient *http.Client
}

func (Manager) Ensure

func (m Manager) Ensure(ctx context.Context, manifest ReleaseManifest) (InstalledRuntime, error)

func (Manager) Rollback

func (m Manager) Rollback() (InstalledRuntime, error)

type ManifestSignature

type ManifestSignature struct {
	KeyID string `json:"key_id"`
	Value string `json:"value"`
}

type ReleaseManifest

type ReleaseManifest struct {
	SchemaVersion int                        `json:"schema_version"`
	Engine        string                     `json:"engine"`
	Version       string                     `json:"version"`
	BuildID       string                     `json:"build_id"`
	Artifacts     map[string]RuntimeArtifact `json:"artifacts"`
	Signature     ManifestSignature          `json:"signature"`
}

func LoadReleaseManifest

func LoadReleaseManifest(path string) (ReleaseManifest, error)

func ParseReleaseManifest

func ParseReleaseManifest(data []byte, source string) (ReleaseManifest, error)

ParseReleaseManifest decodes a runtime release manifest from bytes; source is used only for error messages.

func (ReleaseManifest) SelectArtifact

func (m ReleaseManifest) SelectArtifact(platformKey, vendor string) (RuntimeArtifact, string, error)

SelectArtifact resolves the artifact for a platform and GPU vendor, returning the artifact and the key it was found under.

func (ReleaseManifest) UnsignedBytes

func (m ReleaseManifest) UnsignedBytes() ([]byte, error)

func (ReleaseManifest) VerifySignature

func (m ReleaseManifest) VerifySignature(publicKey ed25519.PublicKey) error

type RunningRuntime

type RunningRuntime struct {
	BaseURL string
	Port    int
	// contains filtered or unexported fields
}

func StartLlamaServer

func StartLlamaServer(ctx context.Context, cfg LaunchConfig) (*RunningRuntime, error)

func (*RunningRuntime) Stop

func (r *RunningRuntime) Stop(ctx context.Context, timeout time.Duration) error

type RuntimeArtifact

type RuntimeArtifact struct {
	URL              string            `json:"url"`
	SHA256           string            `json:"sha256"`
	SizeBytes        int64             `json:"size_bytes"`
	ArchiveType      string            `json:"archive_type"`
	ExecutablePath   string            `json:"executable_path"`
	SupportArtifacts []SupportArtifact `json:"support_artifacts,omitempty"`
}

type SupportArtifact

type SupportArtifact struct {
	Name        string `json:"name"`
	URL         string `json:"url"`
	SHA256      string `json:"sha256"`
	SizeBytes   int64  `json:"size_bytes"`
	ArchiveType string `json:"archive_type"`
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL