logqueue

package
v0.0.18 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 22, 2026 License: MIT Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source
const (
	QueueMaxEntriesKey   = "log.delivery_queue_max_entries"
	QueueMaxBytesKey     = "log.delivery_queue_max_bytes"
	DeliveryBatchSizeKey = "log.delivery_batch_size"
	BackoffMaxSecondsKey = "log.delivery_backoff_max_seconds"

	DefaultQueueMaxEntries   = 10000
	DefaultQueueMaxBytes     = int64(128 << 20)
	DefaultDeliveryBatchSize = 100
	DefaultBackoffMaxSeconds = 60

	MinQueueEntries = 100
	MaxQueueEntries = 1_000_000
	MinQueueBytes   = int64(1 << 20)
	MaxQueueBytes   = int64(8 << 30)
	MinBatchSize    = 1
	MaxBatchSize    = 1000
	MinBackoffSecs  = 1
	MaxBackoffSecs  = 3600
)

Variables

View Source
var (
	ErrInvalidLogBatch      = errors.New("invalid log batch")
	ErrLogPersistenceFailed = errors.New("log persistence failed")
)

Functions

func BatchSize

func BatchSize(batch LogBatch) int64

func DeliverySettingDefinitions

func DeliverySettingDefinitions() map[string]DeliverySettingDefinition

Types

type CoreSettingsFinder

type CoreSettingsFinder struct {
	// contains filtered or unexported fields
}

func NewCoreSettingsFinder

func NewCoreSettingsFinder(dbFinder func() *gorm.DB, onError func(error)) *CoreSettingsFinder

func (*CoreSettingsFinder) Find

type DatabaseConnector

type DatabaseConnector interface {
	Open(context.Context) (*gorm.DB, error)
}

type DatabaseConnectorFunc

type DatabaseConnectorFunc func(context.Context) (*gorm.DB, error)

func (DatabaseConnectorFunc) Open

type DeliveryMetrics

type DeliveryMetrics interface {
	Observe(deliveryqueue.Stats, bool)
	BacklogCleared(uint64, uint64)
	WriteFailure()
	SnapshotFailure()
}

type DeliverySettingDefinition

type DeliverySettingDefinition struct {
	Default string
	Min     int64
	Max     int64
}

type DeliverySettings

type DeliverySettings struct {
	QueueMaxEntries   int
	QueueMaxBytes     int64
	DeliveryBatchSize int
	BackoffMaxSeconds int
}

func DefaultDeliverySettings

func DefaultDeliverySettings() DeliverySettings

type DeliveryStatus

type DeliveryStatus struct {
	Queue       deliveryqueue.Stats
	SchemaReady bool
	LastError   string
}

type HistoricalRequestBatchWriter

type HistoricalRequestBatchWriter struct {
	DBFinder func() *gorm.DB
}

func (*HistoricalRequestBatchWriter) MergeAggregates

func (*HistoricalRequestBatchWriter) ReplaceAggregates

func (*HistoricalRequestBatchWriter) WriteHistory

func (w *HistoricalRequestBatchWriter) WriteHistory(ctx context.Context, requests []models.RequestLog, traces []models.RequestTrace) error

func (*HistoricalRequestBatchWriter) WriteRequests

func (w *HistoricalRequestBatchWriter) WriteRequests(
	ctx context.Context, cursorKey string, lastSourceID uint, rows []models.RequestLog,
) error

func (*HistoricalRequestBatchWriter) WriteTraces

func (w *HistoricalRequestBatchWriter) WriteTraces(
	ctx context.Context, cursorKey string, lastSourceID uint, rows []models.RequestTrace,
) error

type LogBatch

type LogBatch struct {
	Request      models.RequestLog               `json:"request"`
	Traces       []models.RequestTrace           `json:"traces,omitempty"`
	TokenDaily   []models.TokenDailyBilling      `json:"token_daily,omitempty"`
	ChannelDaily []models.ChannelDailyBilling    `json:"channel_daily,omitempty"`
	Hourly       []models.UsageHourlyBucket      `json:"hourly,omitempty"`
	Duration     []models.UsageDurationHistogram `json:"duration,omitempty"`
	TTFT         []models.UsageTTFTHistogram     `json:"ttft,omitempty"`
	TPS          []models.UsageTPSHistogram      `json:"tps,omitempty"`
	UserTTFT     []models.UsageUserTTFTHistogram `json:"user_ttft,omitempty"`
	UserTPS      []models.UsageUserTPSHistogram  `json:"user_tps,omitempty"`
	APIRequest   *models.APIRequestLog           `json:"api_request,omitempty"`
	APITraces    []models.APIRequestTrace        `json:"api_traces,omitempty"`
}

LogBatch is the complete non-billing write unit for one request. The request_id unique key makes replay safe; aggregate deltas are applied only when the request row is new.

func BuildRequestAggregateBatch

func BuildRequestAggregateBatch(log models.UsageLog) LogBatch

BuildRequestAggregateBatch returns the request row and every aggregate delta derived from it. Trace rows remain the caller's responsibility.

type LogBatchWriter

type LogBatchWriter struct {
	DBFinder func() *gorm.DB
}

func (*LogBatchWriter) Write

func (w *LogBatchWriter) Write(ctx context.Context, batches []LogBatch) error

type LogDeliveryWorker

type LogDeliveryWorker struct {
	// contains filtered or unexported fields
}

func NewLogDeliveryWorker

func NewLogDeliveryWorker(options WorkerOptions) *LogDeliveryWorker

func (*LogDeliveryWorker) ClearBacklog

func (w *LogDeliveryWorker) ClearBacklog() (deliveryqueue.ClearResult, error)

func (*LogDeliveryWorker) Enqueue

func (*LogDeliveryWorker) Queue

func (*LogDeliveryWorker) RetryNow

func (w *LogDeliveryWorker) RetryNow()

func (*LogDeliveryWorker) SchemaReady

func (w *LogDeliveryWorker) SchemaReady() bool

func (*LogDeliveryWorker) Start

func (w *LogDeliveryWorker) Start(parent context.Context) error

func (*LogDeliveryWorker) Status

func (w *LogDeliveryWorker) Status() DeliveryStatus

func (*LogDeliveryWorker) Stop

func (w *LogDeliveryWorker) Stop(ctx context.Context) error

type SQLiteLogConnector

type SQLiteLogConnector struct {
	Path      string
	Connector *masterdatabase.Connector
}

func (SQLiteLogConnector) Open

type SettingsFinder

type SettingsFinder interface {
	Find(context.Context) DeliverySettings
}

type WorkerOptions

type WorkerOptions struct {
	Queue        *deliveryqueue.Queue[LogBatch]
	Writer       *LogBatchWriter
	Settings     SettingsFinder
	Connector    DatabaseConnector
	Handoff      func(*gorm.DB) *gorm.DB
	Metrics      DeliveryMetrics
	SnapshotPath string

	PollInterval     time.Duration
	SettingsInterval time.Duration
	SnapshotInterval time.Duration
	FlushTimeout     time.Duration
	RetryBase        time.Duration
	RetryMax         time.Duration
	OnError          func(error)
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL