operator

package
v0.1.2-alpha Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 31, 2026 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ActionNodeDrain      = "node.drain"
	ActionNodePause      = "node.pause"
	ActionNodeQuarantine = "node.quarantine"
	ActionJobRetry       = "job.retry"
	ActionJobCancel      = "job.cancel"
	ActionPayoutCreate   = "payout.create"
	ActionPayoutExport   = "payout.export"
	ActionPayoutConfirm  = "payout.confirm"
	ActionCatalogSync    = "catalog.sync"
	ActionCreditsRelease = "credits.release"
	ActionFleetSetRegion = "fleet.set_region"
	ActionNodeSetRegion  = "node.set_region"
)
View Source
const (
	ListingLive     = "live"
	ListingWaitlist = "waitlist"
	ListingHidden   = "hidden"
)

Public listing statuses mirror the catalog manifest listing block and the models.listing_status column.

View Source
const (
	AvailabilityAvailable = "available"
	AvailabilityLimited   = "limited"
	AvailabilityOffline   = "offline"
	AvailabilityWaitlist  = "waitlist"
)

Public availability states. Only the first three describe measured supply; waitlist means the model is offered for applications with no supply yet.

Variables

This section is empty.

Functions

func EffectiveRegion

func EffectiveRegion(nodeRegion, fleetRegion string) string

func NormalizeEmail

func NormalizeEmail(email string) string

func NormalizeRegion

func NormalizeRegion(value string) (string, error)

func ValidateExpectedVolume

func ValidateExpectedVolume(value string) error

ValidateExpectedVolume accepts an empty band or one of the published monthly output volume options.

Types

type Alert

type Alert struct {
	Code      string         `json:"code"`
	Severity  string         `json:"severity"`
	Message   string         `json:"message"`
	Metadata  map[string]any `json:"metadata,omitempty"`
	CreatedAt time.Time      `json:"created_at"`
}

func BuildAlerts

func BuildAlerts(counts AlertCounts, cfg AlertConfig, now time.Time) []Alert

type AlertConfig

type AlertConfig struct {
	DisconnectSpikeWindow     time.Duration
	DisconnectSpikeThreshold  int
	JobFailureWindow          time.Duration
	JobFailureRateThreshold   float64
	RuntimeCrashWindow        time.Duration
	RuntimeCrashThreshold     int
	OverTempWindow            time.Duration
	NoCapacityWindow          time.Duration
	AuthAnomalyWindow         time.Duration
	AuthAnomalyThreshold      int
	LedgerImbalanceCheck      bool
	NoCapacityForModelEnabled bool
}

func DefaultAlertConfig

func DefaultAlertConfig() AlertConfig

type AlertCounts

type AlertCounts struct {
	Disconnects      int
	JobsTotal        int
	JobsFailed       int
	HashMismatches   int
	RuntimeCrashes   int
	OverTemps        int
	LedgerImbalances int
	AuthAnomalies    int
	NoCapacityModels []string
}

type AttemptSummary

type AttemptSummary struct {
	ID            string     `json:"id"`
	NodeID        string     `json:"node_id,omitempty"`
	AttemptNumber int        `json:"attempt_number"`
	Status        string     `json:"status"`
	ErrorCode     string     `json:"error_code,omitempty"`
	CreatedAt     time.Time  `json:"created_at"`
	AcceptedAt    *time.Time `json:"accepted_at,omitempty"`
	StartedAt     *time.Time `json:"started_at,omitempty"`
	FinishedAt    *time.Time `json:"finished_at,omitempty"`
}

type AuditEvent

type AuditEvent struct {
	ID         string          `json:"id"`
	ActorType  string          `json:"actor_type"`
	ActorID    string          `json:"actor_id,omitempty"`
	Action     string          `json:"action"`
	TargetType string          `json:"target_type,omitempty"`
	TargetID   string          `json:"target_id,omitempty"`
	Metadata   json.RawMessage `json:"metadata"`
	CreatedAt  time.Time       `json:"created_at"`
}

type AuditLog

type AuditLog struct {
	OperatorActions []OperatorAction `json:"operator_actions"`
	SecurityEvents  []SecurityEvent  `json:"security_events"`
	ManifestChanges []AuditEvent     `json:"manifest_changes"`
}

type ContributionCard

type ContributionCard struct {
	NodeID                   string    `json:"node_id"`
	NodeName                 string    `json:"node_name"`
	NightsActive             int64     `json:"nights_active"`
	JobsAccepted             int64     `json:"jobs_accepted"`
	TokensServed             int64     `json:"tokens_served"`
	CreditEarnedMicrodollars int64     `json:"credit_earned_microdollars"`
	GeneratedAt              time.Time `json:"generated_at"`
}

type Fleet

type Fleet struct {
	ID               string    `json:"id"`
	OrganizationID   string    `json:"organization_id"`
	Name             string    `json:"name"`
	Region           string    `json:"region,omitempty"`
	ScheduleFrom     string    `json:"schedule_from,omitempty"`
	ScheduleUntil    string    `json:"schedule_until,omitempty"`
	ScheduleTimezone string    `json:"schedule_timezone"`
	CreatedAt        time.Time `json:"created_at"`
}

type HardwareProfile

type HardwareProfile struct {
	HardwareClass    string  `json:"hardware_class"`
	MinVRAMMB        int     `json:"min_vram_mb"`
	MinRAMMB         int     `json:"min_ram_mb"`
	TokensPerSecond  float64 `json:"tokens_per_second,omitempty"`
	MaxContextTokens int     `json:"max_context_tokens"`
}

type JobDetail

type JobDetail struct {
	JobSummary
	AttemptsDetail []AttemptSummary `json:"attempts_detail"`
	Usage          *protocol.Usage  `json:"usage,omitempty"`
}

type JobSummary

type JobSummary struct {
	ID             string     `json:"id"`
	OrganizationID string     `json:"organization_id,omitempty"`
	ModelID        string     `json:"model_id"`
	State          string     `json:"state"`
	Priority       string     `json:"priority"`
	Attempts       int        `json:"attempts"`
	ErrorCode      string     `json:"error_code,omitempty"`
	CreatedAt      time.Time  `json:"created_at"`
	UpdatedAt      time.Time  `json:"updated_at"`
	CompletedAt    *time.Time `json:"completed_at,omitempty"`
	DeadlineAt     *time.Time `json:"deadline_at,omitempty"`
	Timings        JobTimings `json:"timings"`
}

type JobTimings

type JobTimings struct {
	QueueMilliseconds *int64 `json:"queue_milliseconds,omitempty"`
	RunMilliseconds   *int64 `json:"run_milliseconds,omitempty"`
	TotalMilliseconds *int64 `json:"total_milliseconds,omitempty"`
}

type LedgerOverview

type LedgerOverview struct {
	CustomerChargesMicrodollars      int64          `json:"customer_charges_microdollars"`
	HostPendingCreditMicrodollars    int64          `json:"host_pending_credit_microdollars"`
	HostAvailableCreditMicrodollars  int64          `json:"host_available_credit_microdollars"`
	VerificationOverheadMicrodollars int64          `json:"verification_overhead_microdollars"`
	PayoutBatches                    []PayoutRecord `json:"payout_batches"`
}

type ModelAvailability

type ModelAvailability struct {
	ModelID        string `json:"model_id"`
	AvailableNodes int    `json:"available_nodes"`
}

type ModelSummary

type ModelSummary struct {
	ID               string            `json:"id"`
	DisplayName      string            `json:"display_name"`
	Status           string            `json:"status"`
	DataClass        string            `json:"data_class"`
	Version          string            `json:"version"`
	CatalogStatus    string            `json:"catalog_status"`
	Capabilities     []string          `json:"capabilities"`
	Pricing          jobs.ModelPricing `json:"pricing"`
	PriceVersion     string            `json:"price_version"`
	AvailableNodes   int               `json:"available_nodes"`
	HardwareProfiles []HardwareProfile `json:"hardware_profiles"`
}

type NodeReputation

type NodeReputation struct {
	TotalAcceptedJobs         int64   `json:"total_accepted_jobs"`
	RollingSuccessRate        float64 `json:"rolling_success_rate"`
	TimeoutRate               float64 `json:"timeout_rate"`
	HashMismatchCount         int64   `json:"hash_mismatch_count"`
	ChallengePassRate         float64 `json:"challenge_pass_rate"`
	DuplicateDisagreementRate float64 `json:"duplicate_disagreement_rate"`
	SessionStability          float64 `json:"session_stability"`
	LastQuarantineReason      string  `json:"last_quarantine_reason,omitempty"`
}

type NodeSummary

type NodeSummary struct {
	ID                      string             `json:"id"`
	OrganizationID          string             `json:"organization_id,omitempty"`
	FleetID                 string             `json:"fleet_id,omitempty"`
	FleetName               string             `json:"fleet_name,omitempty"`
	Region                  string             `json:"region,omitempty"`
	State                   string             `json:"state"`
	CurrentModelID          string             `json:"current_model_id,omitempty"`
	QuarantinedAt           *time.Time         `json:"quarantined_at,omitempty"`
	LastSeenAt              *time.Time         `json:"last_seen_at,omitempty"`
	SessionStatus           string             `json:"session_status"`
	LastHeartbeatAt         *time.Time         `json:"last_heartbeat_at,omitempty"`
	LastHeartbeatAgeSeconds *int64             `json:"last_heartbeat_age_seconds,omitempty"`
	ScheduleState           string             `json:"schedule_state,omitempty"`
	ThermalState            string             `json:"thermal_state,omitempty"`
	Paused                  bool               `json:"paused"`
	Draining                bool               `json:"draining"`
	GPU                     protocol.GPUStatus `json:"gpu"`
	Reputation              NodeReputation     `json:"reputation"`
	RecentErrors            []string           `json:"recent_errors"`
}

type OperatorAction

type OperatorAction struct {
	ID         string          `json:"id"`
	Action     string          `json:"action"`
	TargetType string          `json:"target_type"`
	TargetID   string          `json:"target_id"`
	Reason     string          `json:"reason,omitempty"`
	Metadata   json.RawMessage `json:"metadata"`
	CreatedAt  time.Time       `json:"created_at"`
}

type Overview

type Overview struct {
	OnlineNodes                     int                 `json:"online_nodes"`
	AvailableByModel                []ModelAvailability `json:"available_by_model"`
	JobsPerHour                     int                 `json:"jobs_per_hour"`
	SuccessRate                     float64             `json:"success_rate"`
	PendingHostCreditMicrodollars   int64               `json:"pending_host_credit_microdollars"`
	AvailableHostCreditMicrodollars int64               `json:"available_host_credit_microdollars"`
	ActiveAlerts                    []Alert             `json:"active_alerts"`
}

type PayoutRecord

type PayoutRecord struct {
	ID                string     `json:"id"`
	OrganizationID    string     `json:"organization_id,omitempty"`
	Status            string     `json:"status"`
	TotalMicrodollars int64      `json:"total_microdollars"`
	CreatedAt         time.Time  `json:"created_at"`
	ExportedAt        *time.Time `json:"exported_at,omitempty"`
	PaidAt            *time.Time `json:"paid_at,omitempty"`
}

type PublicModelAttribution

type PublicModelAttribution struct {
	DisplayText string `json:"display_text"`
	NoticeText  string `json:"notice_text,omitempty"`
	LicenseURL  string `json:"license_url,omitempty"`
	AUPURL      string `json:"aup_url,omitempty"`
}

PublicModelAttribution carries license-mandated attribution for public surfaces (e.g. "Built with Llama"), present only when the catalog manifest declares it.

type PublicModelAvailability

type PublicModelAvailability struct {
	AvailableNodes int    `json:"available_nodes"`
	State          string `json:"state"`
}

type PublicModelLimits

type PublicModelLimits struct {
	ContextTokens   int `json:"context_tokens"`
	MaxOutputTokens int `json:"max_output_tokens"`
}

type PublicModelMarketComparison

type PublicModelMarketComparison struct {
	TypicalInputPerMillionMicrodollars  int64  `json:"typical_input_per_million_microdollars"`
	TypicalOutputPerMillionMicrodollars int64  `json:"typical_output_per_million_microdollars"`
	SourceNote                          string `json:"source_note"`
}

PublicModelMarketComparison carries the operator-recorded typical hosted price for the same model class. It is only ever present when a catalog manifest supplies both numbers and a source note.

type PublicModelPrice

type PublicModelPrice struct {
	InputPerMillionMicrodollars  int64 `json:"input_per_million_microdollars"`
	OutputPerMillionMicrodollars int64 `json:"output_per_million_microdollars"`
}

type PublicModelStatus

type PublicModelStatus struct {
	ModelID                       string                       `json:"model_id"`
	DisplayName                   string                       `json:"display_name"`
	Description                   string                       `json:"description"`
	ListingStatus                 string                       `json:"listing_status"`
	Capabilities                  []string                     `json:"capabilities"`
	Price                         PublicModelPrice             `json:"price"`
	MarketComparison              *PublicModelMarketComparison `json:"market_comparison"`
	Attribution                   *PublicModelAttribution      `json:"attribution,omitempty"`
	DataClass                     string                       `json:"data_class"`
	Limits                        PublicModelLimits            `json:"limits"`
	Availability                  PublicModelAvailability      `json:"availability"`
	TypicalOutputTokensPerSecond  *float64                     `json:"typical_output_tokens_per_second"`
	ExpectedOutputTokensPerSecond *float64                     `json:"expected_output_tokens_per_second"`
	Regions                       []string                     `json:"regions"`
	Version                       string                       `json:"version"`
}

type PublicStatus

type PublicStatus struct {
	ConnectedNodeCount        int                 `json:"connected_node_count"`
	Cities                    []string            `json:"cities"`
	ModelsAvailable           []ModelAvailability `json:"models_available"`
	Models                    []PublicModelStatus `json:"models"`
	RegionsOnline             []string            `json:"regions_online"`
	RequesterRegion           *string             `json:"requester_region"`
	JobsCompleted24h          int64               `json:"jobs_completed_24h"`
	JobsCompletedTotal        int64               `json:"jobs_completed_total"`
	OutputTokensServed24h     int64               `json:"output_tokens_served_24h"`
	OutputTokensServedTotal   int64               `json:"output_tokens_served_total"`
	EstimatedGPUHoursReused   float64             `json:"estimated_gpu_hours_reused"`
	EstimatedGPUHoursReused24 float64             `json:"estimated_gpu_hours_reused_24h"`
	GeneratedAt               time.Time           `json:"generated_at"`
}

type ScheduleDefaults

type ScheduleDefaults struct {
	From     string
	Until    string
	Timezone string
	Source   string
}

func EffectiveSchedule

func EffectiveSchedule(local ScheduleDefaults, hasLocalOverride bool, fleet ScheduleDefaults) ScheduleDefaults

type SecurityEvent

type SecurityEvent struct {
	ID             string          `json:"id"`
	Severity       string          `json:"severity"`
	EventType      string          `json:"event_type"`
	NodeID         string          `json:"node_id,omitempty"`
	OrganizationID string          `json:"organization_id,omitempty"`
	Metadata       json.RawMessage `json:"metadata"`
	CreatedAt      time.Time       `json:"created_at"`
}

type Store

type Store struct {
	Pool        *pgxpool.Pool
	JobStore    jobs.PGStore
	LedgerStore ledger.Store
	Alerts      AlertConfig
	StaleAfter  time.Duration
	Now         func() time.Time
}

func (Store) AlertsList

func (s Store) AlertsList(ctx context.Context) ([]Alert, error)

func (Store) Audit

func (s Store) Audit(ctx context.Context) (AuditLog, error)

func (Store) CancelJob

func (s Store) CancelJob(ctx context.Context, jobID, reason string, now time.Time) error

func (Store) ConfirmPayoutBatch

func (s Store) ConfirmPayoutBatch(ctx context.Context, batchID string, csvBody []byte, now time.Time) (ledger.PayoutBatch, error)

func (Store) ContributionCard

func (s Store) ContributionCard(ctx context.Context, nodeID string, now time.Time) (ContributionCard, error)

func (Store) CreateFleet

func (s Store) CreateFleet(ctx context.Context, orgID, name string, schedule ScheduleDefaults, region string, now time.Time) (Fleet, error)

func (Store) CreatePayoutBatch

func (s Store) CreatePayoutBatch(ctx context.Context, orgID string, now time.Time) (ledger.PayoutBatch, error)

func (Store) DrainNode

func (s Store) DrainNode(ctx context.Context, nodeID, reason string, now time.Time) error

func (Store) ExportPayoutBatch

func (s Store) ExportPayoutBatch(ctx context.Context, batchID string, now time.Time) ([]byte, ledger.PayoutBatch, error)

func (Store) FleetReportCSV

func (s Store) FleetReportCSV(ctx context.Context, fleetID string, from, until time.Time) ([]byte, error)

func (Store) JobDetail

func (s Store) JobDetail(ctx context.Context, jobID string) (JobDetail, error)

func (Store) LedgerOverview

func (s Store) LedgerOverview(ctx context.Context) (LedgerOverview, error)

func (Store) ListJobs

func (s Store) ListJobs(ctx context.Context) ([]JobSummary, error)

func (Store) ListModels

func (s Store) ListModels(ctx context.Context) ([]ModelSummary, error)

func (Store) ListNodes

func (s Store) ListNodes(ctx context.Context) ([]NodeSummary, error)

func (Store) ListWaitlist

func (s Store) ListWaitlist(ctx context.Context) ([]WaitlistSignup, error)

func (Store) NodeDetail

func (s Store) NodeDetail(ctx context.Context, nodeID string) (NodeSummary, error)

func (Store) Overview

func (s Store) Overview(ctx context.Context) (Overview, error)

func (Store) PauseNode

func (s Store) PauseNode(ctx context.Context, nodeID, reason string, now time.Time) error

func (Store) PromoteCredits

func (s Store) PromoteCredits(ctx context.Context, now time.Time) (int64, error)

func (Store) PublicStatus

func (s Store) PublicStatus(ctx context.Context, now time.Time) (PublicStatus, error)

func (Store) QuarantineNode

func (s Store) QuarantineNode(ctx context.Context, nodeID, reason string, now time.Time) error

func (Store) RecordManifestSync

func (s Store) RecordManifestSync(ctx context.Context, catalogDir string, synced int, now time.Time) error

func (Store) RecordOperatorAction

func (s Store) RecordOperatorAction(ctx context.Context, action, targetType, targetID, reason string, metadata map[string]any, now time.Time) error

func (Store) RetryJob

func (s Store) RetryJob(ctx context.Context, jobID, reason string, now time.Time) error

func (Store) SetFleetRegion

func (s Store) SetFleetRegion(ctx context.Context, fleetID, region, reason string, now time.Time) error

func (Store) SetNodeRegion

func (s Store) SetNodeRegion(ctx context.Context, nodeID, region, reason string, now time.Time) error

func (Store) SubmitWaitlistApplication

func (s Store) SubmitWaitlistApplication(ctx context.Context, application WaitlistApplication, now time.Time) (WaitlistSignup, bool, error)

SubmitWaitlistApplication upserts an access application on (email, model_id). Applications are resubmittable: a returning applicant with a better use case must overwrite their previous answers rather than be dropped, and the same person may apply for several models. Reports whether the row was newly inserted.

func (Store) WaitlistCSV

func (s Store) WaitlistCSV(ctx context.Context) ([]byte, error)

type WaitlistApplication

type WaitlistApplication struct {
	Email          string
	Name           string
	UseCase        string
	ExpectedVolume string
	DataAck        bool
	ModelID        string
	Source         string
}

WaitlistApplication is one manually reviewed request for developer access.

type WaitlistSignup

type WaitlistSignup struct {
	ID             string    `json:"id"`
	Email          string    `json:"email"`
	Name           string    `json:"name,omitempty"`
	UseCase        string    `json:"use_case,omitempty"`
	ExpectedVolume string    `json:"expected_volume,omitempty"`
	DataAck        bool      `json:"data_ack"`
	ModelID        string    `json:"model_id,omitempty"`
	Source         string    `json:"source"`
	CreatedAt      time.Time `json:"created_at"`
	LastAppliedAt  time.Time `json:"last_applied_at"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL