wal

package
v0.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 10, 2026 License: MIT Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source
const DefaultIOBufSize = 4 * 1024

DefaultIOBufSize is the default buffer size for bufio.Writer.

Variables

View Source
var (
	ErrNotWALSegment     = errors.New("not a WAL segment")
	ErrInvalidWALSegment = errors.New("invalid WAL segment")
)
View Source
var (
	ErrMaxDiskUsageExceeded   = fmt.Errorf("max disk usage exceeded")
	ErrMaxSegmentsExceeded    = fmt.Errorf("max segments exceeded")
	ErrMaxSegmentSizeExceeded = fmt.Errorf("max segment size exceeded")
	ErrSegmentClosed          = fmt.Errorf("segment closed")
	ErrSegmentLocked          = fmt.Errorf("segment locked")
)

Functions

func Filename

func Filename(database, table, schema, epoch string) string

func HasSampleMetadata

func HasSampleMetadata(b []byte) bool

func IsSegment

func IsSegment(path string) bool

IsSegment returns true if the file is a valid segment file.

func NewSegmentMerger

func NewSegmentMerger(paths ...string) (io.ReadCloser, error)

func ParseFilename

func ParseFilename(path string) (database, table, schema, epoch string, err error)

func ParsePrefix added in v0.4.0

func ParsePrefix(prefix string) (database, table string, err error)

ParsePrefix returns the database and table of a WAL prefix in the form <database>_<table> or <database>_<table>_<schema>.

func WithSkipHeader

func WithSkipHeader(r *SegmentReader)

Types

type Admission added in v0.4.0

type Admission struct {
	// contains filtered or unexported fields
}

Admission reserves disk capacity before writes. All values are physical WAL bytes, including buffered writes and headers; rotation does not change usage.

func NewAdmission added in v0.4.0

func NewAdmission(opts AdmissionOpts) *Admission

func (*Admission) Enforced added in v0.4.0

func (a *Admission) Enforced() bool

Enforced reports whether table budgets reject writes. When false they are only measured, so callers must not turn a table budget into a hard limit.

func (*Admission) Fits added in v0.4.0

func (a *Admission) Fits(d Destination, bytes int64) bool

Fits reports whether bytes more fit in the global byte budget available to d.

func (*Admission) FitsAll added in v0.4.0

func (a *Admission) FitsAll(bytes int64) bool

FitsAll reports whether bytes more fit in the global byte budget of every destination.

func (*Admission) FitsEstimate added in v0.4.0

func (a *Admission) FitsEstimate(d Destination, bytes int64) bool

FitsEstimate is like Fits for an upper-bound estimate of the bytes. An estimate larger than the whole limit is taken as the limit, so a write whose estimate could never pass the check still fits in an empty store, and is then checked against its actual size, rather than being rejected on every retry.

func (*Admission) FitsTable added in v0.4.0

func (a *Admission) FitsTable(d Destination, bytes int64) bool

FitsTable reports whether bytes more fit in d's enforced table byte budget. Unlike Reserve it does not throttle the table when they do not, so a caller can split a write that is too large without the rejected whole throttling the table against the smaller parts that fit.

func (*Admission) Forget added in v0.4.0

func (a *Admission) Forget(d Destination, path string)

Forget must only be called after a file has been removed successfully.

func (*Admission) Limit added in v0.4.0

func (a *Admission) Limit(d Destination) Usage

func (*Admission) RecordRejection added in v0.4.0

func (a *Admission) RecordRejection(d Destination, records, bytes int64) bool

RecordRejection accounts for records shed at a table budget and reports whether this rejection should be logged, rate limited to once per minute.

func (*Admission) Reserve added in v0.4.0

func (a *Admission) Reserve(path string, d Destination, bytes, segments int64) (*Reservation, error)

func (*Admission) Restore added in v0.4.0

func (a *Admission) Restore(path string, d Destination, bytes int64)

Restore registers an existing, repaired segment before intake starts.

func (*Admission) Snapshot added in v0.4.0

func (a *Admission) Snapshot() map[Destination]Usage

func (*Admission) Status added in v0.4.0

func (a *Admission) Status() map[Destination]TableStatus

func (*Admission) Table added in v0.4.0

func (a *Admission) Table(d Destination) *TableBudget

Table returns the budget for a destination, creating it on first use. The returned handle is stable and should be cached by writers.

func (*Admission) Throttled added in v0.4.0

func (a *Admission) Throttled(d Destination) (string, bool)

Throttled reports whether a destination is currently over budget and enforcement is active. It is a cheap pre-admission check that lets callers skip encoding and WAL creation for work that will be rejected anyway.

func (*Admission) Tracks added in v0.4.0

func (a *Admission) Tracks(path string) bool

Tracks returns true if path is a segment file whose capacity is accounted for. It does not create a table budget for an unknown destination.

func (*Admission) Usage added in v0.4.0

func (a *Admission) Usage() Usage

Usage returns the exact totals across every table. It is a reporting path, not a write path, so it sums per-table state rather than reading the batched global estimate.

func (*Admission) UsesReservation added in v0.4.0

func (a *Admission) UsesReservation(d Destination) bool

UsesReservation reports whether d may write into the realtime reservation, so its global limit is the whole budget rather than the queued share.

type AdmissionOpts added in v0.4.0

type AdmissionOpts struct {
	Global    Usage
	Default   Usage
	Overrides []TableLimit

	// Enforce rejects writes that exceed a table budget. When false the budget
	// is still measured and reported, which lets operators size limits from
	// real traffic before any data is shed.
	Enforce bool

	// RealtimeReservedBytes is the part of Global.Bytes that only realtime
	// tables may use, so queued backlog cannot consume the space realtime
	// ingestion depends on. It applies in both enforced and observe-only
	// modes, and only when Policy has realtime tables.
	RealtimeReservedBytes int64
	Policy                *ingestpolicy.Policy
}

func ResolveAdmission added in v0.4.0

func ResolveAdmission(global Usage, maxBytes, maxSegments int64, disabled bool, overrides []TableLimit) (AdmissionOpts, error)

ResolveAdmission applies per-instance defaults. Zero means 10% of the global capacity; disabling defaults still permits explicit table overrides.

func ResolveAdmissionWithEnforcement added in v0.4.0

func ResolveAdmissionWithEnforcement(global Usage, maxBytes, maxSegments int64, disabled, enforce bool, overrides []TableLimit) (AdmissionOpts, error)

ResolveAdmissionWithEnforcement resolves budgets and selects whether exceeding a table budget rejects writes or is only reported.

type Destination added in v0.4.0

type Destination struct {
	Database string
	Table    string
}

Destination identifies a normalized storage table, independently of its schema.

func DestinationFromFilename added in v0.4.0

func DestinationFromFilename(path string) (Destination, error)

func DestinationFromPrefix added in v0.4.0

func DestinationFromPrefix(prefix string) (Destination, error)

func (Destination) String added in v0.4.0

func (d Destination) String() string

type File

type File struct {
	Path     string
	Database string
	Table    string
	Schema   string
	Epoch    string
	Key      string
}

type Index

type Index struct {

	// Admission owns live capacity accounting and is notified when a segment is
	// removed so its budget is released.
	Admission *Admission
	// contains filtered or unexported fields
}

Index provides overview of all segments in a repository.

The index owns closed-segment bookkeeping only. Live capacity accounting is owned by Admission; the index notifies it when a segment is removed so the two never maintain parallel counters that can diverge.

func NewIndex

func NewIndex() *Index

NewIndex returns a new index.

func (*Index) Add

func (i *Index) Add(s SegmentInfo)

Add adds a closed segment to the index and notifies subscribers.

func (*Index) Contains added in v0.4.0

func (i *Index) Contains(path string) bool

Contains returns true if the index has a segment with path.

func (*Index) Get

func (i *Index) Get(infos []SegmentInfo, prefix string) []SegmentInfo

Get returns all segments for a given prefix.

func (*Index) LargestCountPrefix

func (i *Index) LargestCountPrefix() string

LargestCountPrefix returns the prefix of the segment with the largest total count.

func (*Index) LargestSizePrefix

func (i *Index) LargestSizePrefix() string

LargestSizePrefix returns the prefix of the segment with the largest total size.

func (*Index) Oldest added in v0.4.0

func (i *Index) Oldest(n int) []SegmentInfo

Oldest returns up to n of the oldest segments in the index, oldest first.

func (*Index) OldestPrefix

func (i *Index) OldestPrefix() string

Oldest returns the prefix of the oldest segment.

func (*Index) OldestSegmentAge added in v0.2.0

func (i *Index) OldestSegmentAge() time.Duration

OldestSegmentAge returns the age of the oldest segment in the index or 0 if the index is empty.

func (*Index) Prefixes added in v0.4.0

func (i *Index) Prefixes(dst []string) []string

Prefixes appends all prefixes to dst in no particular order and returns it.

func (*Index) PrefixesByAge

func (i *Index) PrefixesByAge() []string

PrefixesByAge returns all prefixes sorted by oldest to newest.

func (*Index) PrefixesByCount

func (i *Index) PrefixesByCount() []string

PrefixesByCount returns all prefixes sorted by total count least to greatest. If there is a tie, the prefix that is lexicographically first is returned.

func (*Index) PrefixesBySize

func (i *Index) PrefixesBySize() []string

PrefixesBySize returns all prefixes sorted by total size least to greatest.

func (*Index) Remove

func (i *Index) Remove(s SegmentInfo)

Remove removes a segment from the index and releases the capacity it held.

func (*Index) Subscribe added in v0.4.0

func (i *Index) Subscribe(fn func(SegmentInfo)) (unsubscribe func())

Subscribe registers fn to be called each time a closed segment is added to the index. fn is called synchronously by the goroutine that added the segment, after the index lock is released, so it must return quickly and must not block. Segments added before Subscribe are not replayed; callers should read the index after subscribing to discover existing segments. The returned func unsubscribes fn.

func (*Index) TotalPrefixes

func (i *Index) TotalPrefixes() int

TotalPrefixes returns the total number of prefixes in the index.

func (*Index) TotalSegments

func (i *Index) TotalSegments() int

TotalSegments returns the total number of segments in the index.

func (*Index) TotalSize

func (i *Index) TotalSize() int64

TotalSize returns the total size of all segments in the index.

func (*Index) TotalSizeByPriority added in v0.4.0

func (i *Index) TotalSizeByPriority(p ingestpolicy.Priority) int64

TotalSizeByPriority returns the total size of segments in the index with the given ingestion priority.

func (*Index) WriteDebug added in v0.2.0

func (i *Index) WriteDebug(w io.Writer) error

type Iterator

type Iterator interface {
	Next() (bool, error)
	Value() []byte
	Close() error
	// Verify ensures the Iterator can iterate over a continuous series of blocks without error.
	Verify() (int, error)
	Metadata() (SampleType, uint32)
}

func NewSegmentIterator

func NewSegmentIterator(r io.ReadCloser) (Iterator, error)

type Option

type Option func(*SegmentReader)

type Repository

type Repository struct {
	// contains filtered or unexported fields
}

Repository is a collection of WALs.

func NewRepository

func NewRepository(opts RepositoryOpts) *Repository

func (*Repository) Admission added in v0.4.0

func (s *Repository) Admission() *Admission

func (*Repository) Close

func (s *Repository) Close() error

func (*Repository) Count

func (s *Repository) Count() int

func (*Repository) Get

func (s *Repository) Get(ctx context.Context, key []byte) (*WAL, error)

func (*Repository) Index

func (s *Repository) Index() *Index

func (*Repository) IsActiveSegment added in v0.4.0

func (s *Repository) IsActiveSegment(path string) bool

IsActiveSegment returns true if path is the active, not yet closed, segment of a WAL.

func (*Repository) Keys

func (s *Repository) Keys() [][]byte

func (*Repository) MaxDiskUsage added in v0.4.0

func (s *Repository) MaxDiskUsage(p ingestpolicy.Priority) int64

MaxDiskUsage returns the total disk usage at which writes with the given priority are rejected. Queued writes are limited to MaxDiskUsage minus the realtime reservation so realtime writes can continue when queued data backs up. A value of 0 means no limit.

func (*Repository) Open

func (s *Repository) Open(ctx context.Context) error

func (*Repository) PrefixesByAge

func (s *Repository) PrefixesByAge() []string

func (*Repository) Remove

func (s *Repository) Remove(key []byte) error

func (*Repository) RemoveSegment

func (s *Repository) RemoveSegment(si SegmentInfo)

func (*Repository) Size added in v0.2.0

func (s *Repository) Size() int64

func (*Repository) WriteDebug added in v0.2.0

func (s *Repository) WriteDebug(w io.Writer) error

type RepositoryOpts

type RepositoryOpts struct {
	StorageDir             string
	SegmentMaxSize         int64
	SegmentMaxAge          time.Duration
	StartupOpenConcurrency int

	MaxDiskUsage     int64
	MaxSegmentCount  int
	WALFlushInterval time.Duration
	EnableWALFsync   bool

	// Policy assigns an ingestion priority to each WAL prefix.  When nil, all WALs are queued.
	Policy *ingestpolicy.Policy

	// Realtime overrides the segment rotation settings for realtime WALs.
	Realtime RotationPolicy

	// RealtimeReservedDiskBytes is disk space reserved for realtime WALs.  Queued WALs reject writes once total disk
	// usage reaches MaxDiskUsage minus this value.  It only applies when Policy has realtime tables and is less than
	// MaxDiskUsage.
	RealtimeReservedDiskBytes int64

	// Admission owns per-destination capacity accounting for this repository.
	Admission *Admission
}

type Reservation added in v0.4.0

type Reservation struct {
	// contains filtered or unexported fields
}

func (*Reservation) Finish added in v0.4.0

func (r *Reservation) Finish(bytes int64)

Finish commits the bytes actually written, or rolls back with zero. Segment reservations are committed only when a segment header was created.

func (*Reservation) Release added in v0.4.0

func (r *Reservation) Release()

Release rolls back an unfinished reservation and returns its allocation to the pool. The caller must not access the reservation after Release.

type RotationPolicy added in v0.4.0

type RotationPolicy struct {
	SegmentMaxAge    time.Duration
	SegmentMaxSize   int64
	WALFlushInterval time.Duration
}

RotationPolicy overrides segment rotation settings. Zero values inherit the repository settings.

type SampleType

type SampleType uint16
const (
	UnknownSampleType SampleType = iota
	MetricSampleType
	TraceSampleType
	LogSampleType
)

func SampleMetadata

func SampleMetadata(b []byte) (t SampleType, count uint32)

type Segment

type Segment interface {
	Append(ctx context.Context, buf []byte) (int, error)
	Write(ctx context.Context, buf []byte, opts ...WriteOptions) (int, error)
	Bytes() ([]byte, error)
	Close() error
	ID() string
	Size() int64
	CreatedAt() time.Time
	Reader() (io.ReadCloser, error)
	Path() string

	Iterator() (Iterator, error)
	Info() SegmentInfo
	Flush() error
	// Repair truncates the last bytes in the segment if they are missing, corrupted or have extra data.  This
	// repairs any corrupted segment that may not have fully flushed to disk safely.  The segment is truncated
	// from the first block that is found to be corrupt.
	Repair() error
}

func NewSegment

func NewSegment(dir, prefix string, opts ...SegmentOption) (Segment, error)

func Open

func Open(path string) (Segment, error)

type SegmentInfo

type SegmentInfo struct {
	Prefix    string
	Ulid      string
	Path      string
	Size      int64
	CreatedAt time.Time

	// Priority is the ingestion priority of the segment's table.
	Priority ingestpolicy.Priority
}

type SegmentMerger

type SegmentMerger struct {
	// contains filtered or unexported fields
}

func (*SegmentMerger) Close

func (m *SegmentMerger) Close() error

func (*SegmentMerger) Read

func (m *SegmentMerger) Read(p []byte) (n int, err error)

type SegmentOption

type SegmentOption func(s *segment)

func WithAdmission added in v0.4.0

func WithAdmission(a *Admission) SegmentOption

func WithFlushIntervale

func WithFlushIntervale(d time.Duration) SegmentOption

func WithFsync added in v0.2.0

func WithFsync(b bool) SegmentOption

type SegmentReader

type SegmentReader struct {
	// contains filtered or unexported fields
}

func NewSegmentReader

func NewSegmentReader(path string, opts ...Option) (*SegmentReader, error)

func (*SegmentReader) Close

func (s *SegmentReader) Close() error

func (*SegmentReader) Path added in v0.2.0

func (s *SegmentReader) Path() string

func (*SegmentReader) Read

func (s *SegmentReader) Read(p []byte) (n int, err error)

func (*SegmentReader) SampleMetadata

func (s *SegmentReader) SampleMetadata() (t SampleType, sampleCount uint32)

type SegmentTooLargeError added in v0.4.0

type SegmentTooLargeError struct {
	Destination Destination
	Size        int64
	Limit       int64
}

SegmentTooLargeError reports a payload that cannot fit in a destination's budget even when that destination is completely empty. Unlike TableOverloadedError this is not relieved by draining, so the sender must split the batch or an operator must raise the limit.

func (*SegmentTooLargeError) Error added in v0.4.0

func (e *SegmentTooLargeError) Error() string

type TableBudget added in v0.4.0

type TableBudget struct {
	// contains filtered or unexported fields
}

TableBudget owns the admission state for one destination. Writers resolve it once and reuse it, so the write path touches only this destination's lock and the process-wide counters. Budgets for different tables never contend.

func (*TableBudget) Reserve added in v0.4.0

func (t *TableBudget) Reserve(path string, bytes, segments int64) (*Reservation, error)

Reserve claims capacity for a pending write. The caller must call Finish with the bytes actually written and then Release.

type TableLimit added in v0.4.0

type TableLimit struct {
	Database    string `toml:"database" json:"database"`
	Table       string `toml:"table" json:"table"`
	MaxBytes    int64  `toml:"max-bytes" json:"max-bytes"`
	MaxSegments int64  `toml:"max-segments" json:"max-segments"`
}

TableLimit overrides the default local backlog budget for a normalized table.

func ParseTableLimit added in v0.4.0

func ParseTableLimit(value string) (TableLimit, error)

type TableOverloadedError added in v0.4.0

type TableOverloadedError struct {
	Destination Destination
	Reason      string
	RetryAfter  time.Duration
}

func (*TableOverloadedError) Error added in v0.4.0

func (e *TableOverloadedError) Error() string

type TableStatus added in v0.4.0

type TableStatus struct {
	Usage
	Limit                          Usage
	Throttled                      bool
	Enforced                       bool
	WouldReject                    int64
	RejectedRecords, RejectedBytes int64
	OldestAge                      time.Duration

	// ThrottledFor is how long this destination has been continuously over
	// budget. A brief value is a burst; a sustained one means the destination
	// either needs a larger budget or is genuinely producing more than this
	// instance can drain.
	ThrottledFor time.Duration
}

type Usage added in v0.4.0

type Usage struct {
	Bytes    int64
	Segments int64
}

Usage counts bytes and files, including reservations when used by admission.

type WAL

type WAL struct {
	// contains filtered or unexported fields
}

func NewWAL

func NewWAL(opts WALOpts) (*WAL, error)

func (*WAL) Append

func (w *WAL) Append(ctx context.Context, buf []byte) error

func (*WAL) Close

func (w *WAL) Close() error

func (*WAL) Flush

func (w *WAL) Flush() error

func (*WAL) Open

func (w *WAL) Open(ctx context.Context) error

func (*WAL) Path

func (w *WAL) Path() string

Path returns the path of the active segment.

func (*WAL) Priority added in v0.4.0

func (w *WAL) Priority() ingestpolicy.Priority

Priority returns the ingestion priority of the WAL's table.

func (*WAL) Remove

func (w *WAL) Remove(path string) error

func (*WAL) RemoveAll

func (w *WAL) RemoveAll() error

func (*WAL) Segment

func (w *WAL) Segment() Segment

func (*WAL) Size

func (w *WAL) Size() int

func (*WAL) Write

func (w *WAL) Write(ctx context.Context, buf []byte, opts ...WriteOptions) error

type WALOpts

type WALOpts struct {
	StorageDir string

	// WAL segment prefix
	Prefix string

	// SegmentMaxSize is the max size of a segment in bytes before it will be rotated and compressed.
	SegmentMaxSize int64

	// SegmentMaxAge is the max age of a segment before it will be rotated and compressed.
	SegmentMaxAge time.Duration

	// MaxDiskUsage is the max disk usage of WAL segments allowed before writes should be rejected.
	MaxDiskUsage int64

	// MaxSegmentCount is the max number of segments allowed before writes should be rejected.
	MaxSegmentCount int

	// Index is the index of the WAL segments.
	Index *Index

	// WALFlushInterval is the interval at which the WAL should be flushed.
	WALFlushInterval time.Duration

	// EnableWALFsync enables fsync of the segment after every flush.
	EnableWALFsync bool

	// Priority is the ingestion priority of the WAL's table.
	Priority ingestpolicy.Priority

	// Admission owns per-destination capacity accounting.  When nil, table
	// budgets are not applied to this WAL.
	Admission *Admission
	// contains filtered or unexported fields
}

type WriteOptions

type WriteOptions func([]byte)

func WithSampleMetadata

func WithSampleMetadata(t SampleType, count uint32) WriteOptions

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL