runner

package module
v0.0.0-...-92056dc Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 1, 2026 License: Apache-2.0 Imports: 22 Imported by: 0

README

LM runner

Сервис запуска и взаимодействия с LLM

Быстрый старт (сборка и запуск)

apt install build-essential cmake

# Установка зависимостей
make deps

apt install libgomp1

CPU

# Сборка библиотек (без CUDA)
make build-libs-cpu

# Запуск (CPU, без CUDA)
make run-cpu

# Сборка бинарника (CPU)
make build-cpu

GPU

# Сборка библиотек с поддержкой NVIDIA (CUDA)
make build-libs-gpu

# Запуск (GPU, NVIDIA CUDA)
make run-gpu

# Сборка бинарника (CUDA)
make build-gpu

Настройка окружения

export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH
# Обязательная настройка перед запуском lm-runner
# Указывает загрузчику путь к скомпилированным библиотекам
# Без этого бинарник не стартует (ошибка: cannot open shared object file)

Скачивание модели (Hugging Face)

./build/lm-runner download --repo <org/model> --list
./build/lm-runner download --repo <org/model> --file ....gguf

Запуск модель/Загрузка модели в память

# список доступных моделей
./build/lm-runner remote models

# запуск модели
./build/lm-runner remote load --model <name>

# вывод запущенных
./build/lm-runner remote ps

Клиент к запущенному раннеру

./build/lm-runner remote ping
./build/lm-runner remote run --prompt "привет"

# Собрать yaml из Modelfile
./build/lm-runner create myalias -f ./Modelfile [--force]

# Показать yaml манифеста или экспорт в Modelfile
./build/lm-runner show myalias
./build/lm-runner show myalias --modelfile # или -m

# Только путь к .yaml (для скриптов)
./build/lm-runner show myalias --path-only

# Список локальных .gguf в каталоге model_path
./build/lm-runner models

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type InferenceMetrics 

type InferenceMetrics struct {
	// contains filtered or unexported fields
}

func NewInferenceMetrics 

func NewInferenceMetrics() *InferenceMetrics

func (*InferenceMetrics) Get 

func (m *InferenceMetrics) Get() (tokens int64, latencyMs float64, tokensPerSec float64, ttftMs float64)

func (*InferenceMetrics) Record 

func (m *InferenceMetrics) Record(tokens int64, duration time.Duration, ttft time.Duration)

type Pool 

type Pool struct {
	// contains filtered or unexported fields
}

func NewPool 

func NewPool(addresses []string) *Pool

func (*Pool) Add 

func (p *Pool) Add(address string)

func (*Pool) CheckConnection 

func (p *Pool) CheckConnection(ctx context.Context) (bool, error)

func (*Pool) GetGpuInfo 

func (p *Pool) GetGpuInfo(ctx context.Context, address string) *llmrunnerpb.GetGpuInfoResponse

func (*Pool) GetModels 

func (p *Pool) GetModels(ctx context.Context) ([]string, error)

func (*Pool) GetRunners 

func (p *Pool) GetRunners(ctx context.Context) []RunnerInfo

func (*Pool) GetServerInfo 

func (p *Pool) GetServerInfo(ctx context.Context, address string) *llmrunnerpb.ServerInfo

func (*Pool) HasActiveRunners 

func (p *Pool) HasActiveRunners() bool

func (*Pool) Remove 

func (p *Pool) Remove(address string)

func (*Pool) SendMessage 

func (p *Pool) SendMessage(ctx context.Context, model string, messages []*domain.AIChatMessage, stopSequences []string, timeoutSeconds int32, genParams *domain.GenerationParams, renderedPrompt string) (chan domain.TextStreamChunk, error)

func (*Pool) SetRunnerEnabled 

func (p *Pool) SetRunnerEnabled(address string, enabled bool)

type RunnerInfo 

type RunnerInfo struct {
	Address   string
	Enabled   bool
	Connected bool
}

type Server 

type Server struct {
	llmrunnerpb.UnimplementedLLMRunnerServiceServer
	// contains filtered or unexported fields
}

func NewServer 

func NewServer(textProvider provider.TextProvider, gpuCollector gpu.Collector, maxConcurrentGenerations int, unloadAfterRPC bool, modelsDir string) *Server

func (*Server) Embed 

func (s *Server) Embed(ctx context.Context, req *llmrunnerpb.EmbedRequest) (*llmrunnerpb.EmbedResponse, error)

func (*Server) EmbedBatch 

func (s *Server) EmbedBatch(ctx context.Context, req *llmrunnerpb.EmbedBatchRequest) (*llmrunnerpb.EmbedBatchResponse, error)

func (*Server) GetGpuInfo 

func (s *Server) GetGpuInfo(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.GetGpuInfoResponse, error)

func (*Server) GetLoadedModel 

func (s *Server) GetLoadedModel(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.GetLoadedModelResponse, error)

func (*Server) GetModels 

func (s *Server) GetModels(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.GetModelsResponse, error)

func (*Server) GetServerInfo 

func (s *Server) GetServerInfo(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.ServerInfo, error)

func (*Server) LoadModel 

func (s *Server) LoadModel(ctx context.Context, req *llmrunnerpb.LoadModelRequest) (*llmrunnerpb.Empty, error)

func (*Server) ResetMemory 

func (s *Server) ResetMemory(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.Empty, error)

ResetMemory полностью выгружает текущую модель и освобождает VRAM/RAM процесса (тот же эффект, что UnloadModel).

func (*Server) RunnerProbe 

func (s *Server) RunnerProbe(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.RunnerProbeResponse, error)

func (*Server) SendMessage 

func (s *Server) SendMessage(req *llmrunnerpb.SendMessageRequest, stream llmrunnerpb.LLMRunnerService_SendMessageServer) error

func (*Server) UnloadModel 

func (s *Server) UnloadModel(ctx context.Context, _ *llmrunnerpb.Empty) (*llmrunnerpb.Empty, error)

type SysInfo 

type SysInfo struct {
	Hostname      string
	OS            string
	Arch          string
	CPUCores      int32
	MemoryTotalMB uint64
}

func CollectSysInfo 

func CollectSysInfo() SysInfo

Source Files

View all Source files

Directories

Path Synopsis
cmd
lm-runner command
pb
llmrunnerpb

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.