Documentation
¶
Index ¶
- Constants
- Variables
- func Filename(database, table, schema, epoch string) string
- func HasSampleMetadata(b []byte) bool
- func IsSegment(path string) bool
- func NewSegmentMerger(paths ...string) (io.ReadCloser, error)
- func ParseFilename(path string) (database, table, schema, epoch string, err error)
- func ParsePrefix(prefix string) (database, table string, err error)
- func WithSkipHeader(r *SegmentReader)
- type Admission
- func (a *Admission) Enforced() bool
- func (a *Admission) Fits(d Destination, bytes int64) bool
- func (a *Admission) FitsAll(bytes int64) bool
- func (a *Admission) FitsEstimate(d Destination, bytes int64) bool
- func (a *Admission) FitsTable(d Destination, bytes int64) bool
- func (a *Admission) Forget(d Destination, path string)
- func (a *Admission) Limit(d Destination) Usage
- func (a *Admission) RecordRejection(d Destination, records, bytes int64) bool
- func (a *Admission) Reserve(path string, d Destination, bytes, segments int64) (*Reservation, error)
- func (a *Admission) Restore(path string, d Destination, bytes int64)
- func (a *Admission) Snapshot() map[Destination]Usage
- func (a *Admission) Status() map[Destination]TableStatus
- func (a *Admission) Table(d Destination) *TableBudget
- func (a *Admission) Throttled(d Destination) (string, bool)
- func (a *Admission) Tracks(path string) bool
- func (a *Admission) Usage() Usage
- func (a *Admission) UsesReservation(d Destination) bool
- type AdmissionOpts
- type Destination
- type File
- type Index
- func (i *Index) Add(s SegmentInfo)
- func (i *Index) Contains(path string) bool
- func (i *Index) Get(infos []SegmentInfo, prefix string) []SegmentInfo
- func (i *Index) LargestCountPrefix() string
- func (i *Index) LargestSizePrefix() string
- func (i *Index) Oldest(n int) []SegmentInfo
- func (i *Index) OldestPrefix() string
- func (i *Index) OldestSegmentAge() time.Duration
- func (i *Index) Prefixes(dst []string) []string
- func (i *Index) PrefixesByAge() []string
- func (i *Index) PrefixesByCount() []string
- func (i *Index) PrefixesBySize() []string
- func (i *Index) Remove(s SegmentInfo)
- func (i *Index) Subscribe(fn func(SegmentInfo)) (unsubscribe func())
- func (i *Index) TotalPrefixes() int
- func (i *Index) TotalSegments() int
- func (i *Index) TotalSize() int64
- func (i *Index) TotalSizeByPriority(p ingestpolicy.Priority) int64
- func (i *Index) WriteDebug(w io.Writer) error
- type Iterator
- type Option
- type Repository
- func (s *Repository) Admission() *Admission
- func (s *Repository) Close() error
- func (s *Repository) Count() int
- func (s *Repository) Get(ctx context.Context, key []byte) (*WAL, error)
- func (s *Repository) Index() *Index
- func (s *Repository) IsActiveSegment(path string) bool
- func (s *Repository) Keys() [][]byte
- func (s *Repository) MaxDiskUsage(p ingestpolicy.Priority) int64
- func (s *Repository) Open(ctx context.Context) error
- func (s *Repository) PrefixesByAge() []string
- func (s *Repository) Remove(key []byte) error
- func (s *Repository) RemoveSegment(si SegmentInfo)
- func (s *Repository) Size() int64
- func (s *Repository) WriteDebug(w io.Writer) error
- type RepositoryOpts
- type Reservation
- type RotationPolicy
- type SampleType
- type Segment
- type SegmentInfo
- type SegmentMerger
- type SegmentOption
- type SegmentReader
- type SegmentTooLargeError
- type TableBudget
- type TableLimit
- type TableOverloadedError
- type TableStatus
- type Usage
- type WAL
- func (w *WAL) Append(ctx context.Context, buf []byte) error
- func (w *WAL) Close() error
- func (w *WAL) Flush() error
- func (w *WAL) Open(ctx context.Context) error
- func (w *WAL) Path() string
- func (w *WAL) Priority() ingestpolicy.Priority
- func (w *WAL) Remove(path string) error
- func (w *WAL) RemoveAll() error
- func (w *WAL) Segment() Segment
- func (w *WAL) SegmentMaxAge() time.Duration
- func (w *WAL) SetSegmentMaxAge(d time.Duration)
- func (w *WAL) Size() int
- func (w *WAL) Write(ctx context.Context, buf []byte, opts ...WriteOptions) error
- type WALOpts
- type WriteOptions
Constants ¶
const DefaultIOBufSize = 4 * 1024
DefaultIOBufSize is the default buffer size for bufio.Writer.
Variables ¶
var ( ErrNotWALSegment = errors.New("not a WAL segment") ErrInvalidWALSegment = errors.New("invalid WAL segment") )
var ( ErrMaxDiskUsageExceeded = fmt.Errorf("max disk usage exceeded") ErrMaxSegmentsExceeded = fmt.Errorf("max segments exceeded") ErrMaxSegmentSizeExceeded = fmt.Errorf("max segment size exceeded") ErrSegmentClosed = fmt.Errorf("segment closed") ErrSegmentLocked = fmt.Errorf("segment locked") )
Functions ¶
func HasSampleMetadata ¶
func NewSegmentMerger ¶
func NewSegmentMerger(paths ...string) (io.ReadCloser, error)
func ParseFilename ¶
func ParsePrefix ¶ added in v0.4.0
ParsePrefix returns the database and table of a WAL prefix in the form <database>_<table> or <database>_<table>_<schema>.
func WithSkipHeader ¶
func WithSkipHeader(r *SegmentReader)
Types ¶
type Admission ¶ added in v0.4.0
type Admission struct {
// contains filtered or unexported fields
}
Admission reserves disk capacity before writes. All values are physical WAL bytes, including buffered writes and headers; rotation does not change usage.
func NewAdmission ¶ added in v0.4.0
func NewAdmission(opts AdmissionOpts) *Admission
func (*Admission) Enforced ¶ added in v0.4.0
Enforced reports whether table budgets reject writes. When false they are only measured, so callers must not turn a table budget into a hard limit.
func (*Admission) Fits ¶ added in v0.4.0
func (a *Admission) Fits(d Destination, bytes int64) bool
Fits reports whether bytes more fit in the global byte budget available to d.
func (*Admission) FitsAll ¶ added in v0.4.0
FitsAll reports whether bytes more fit in the global byte budget of every destination.
func (*Admission) FitsEstimate ¶ added in v0.4.0
func (a *Admission) FitsEstimate(d Destination, bytes int64) bool
FitsEstimate is like Fits for an upper-bound estimate of the bytes. An estimate larger than the whole limit is taken as the limit, so a write whose estimate could never pass the check still fits in an empty store, and is then checked against its actual size, rather than being rejected on every retry.
func (*Admission) FitsTable ¶ added in v0.4.0
func (a *Admission) FitsTable(d Destination, bytes int64) bool
FitsTable reports whether bytes more fit in d's enforced table byte budget. Unlike Reserve it does not throttle the table when they do not, so a caller can split a write that is too large without the rejected whole throttling the table against the smaller parts that fit.
func (*Admission) Forget ¶ added in v0.4.0
func (a *Admission) Forget(d Destination, path string)
Forget must only be called after a file has been removed successfully.
func (*Admission) Limit ¶ added in v0.4.0
func (a *Admission) Limit(d Destination) Usage
func (*Admission) RecordRejection ¶ added in v0.4.0
func (a *Admission) RecordRejection(d Destination, records, bytes int64) bool
RecordRejection accounts for records shed at a table budget and reports whether this rejection should be logged, rate limited to once per minute.
func (*Admission) Reserve ¶ added in v0.4.0
func (a *Admission) Reserve(path string, d Destination, bytes, segments int64) (*Reservation, error)
func (*Admission) Restore ¶ added in v0.4.0
func (a *Admission) Restore(path string, d Destination, bytes int64)
Restore registers an existing, repaired segment before intake starts.
func (*Admission) Snapshot ¶ added in v0.4.0
func (a *Admission) Snapshot() map[Destination]Usage
func (*Admission) Status ¶ added in v0.4.0
func (a *Admission) Status() map[Destination]TableStatus
func (*Admission) Table ¶ added in v0.4.0
func (a *Admission) Table(d Destination) *TableBudget
Table returns the budget for a destination, creating it on first use. The returned handle is stable and should be cached by writers.
func (*Admission) Throttled ¶ added in v0.4.0
func (a *Admission) Throttled(d Destination) (string, bool)
Throttled reports whether a destination is currently over budget and enforcement is active. It is a cheap pre-admission check that lets callers skip encoding and WAL creation for work that will be rejected anyway.
func (*Admission) Tracks ¶ added in v0.4.0
Tracks returns true if path is a segment file whose capacity is accounted for. It does not create a table budget for an unknown destination.
func (*Admission) Usage ¶ added in v0.4.0
Usage returns the exact totals across every table. It is a reporting path, not a write path, so it sums per-table state rather than reading the batched global estimate.
func (*Admission) UsesReservation ¶ added in v0.4.0
func (a *Admission) UsesReservation(d Destination) bool
UsesReservation reports whether d may write into the realtime reservation, so its global limit is the whole budget rather than the queued share.
type AdmissionOpts ¶ added in v0.4.0
type AdmissionOpts struct {
Global Usage
Default Usage
Overrides []TableLimit
// Enforce rejects writes that exceed a table budget. When false the budget
// is still measured and reported, which lets operators size limits from
// real traffic before any data is shed.
Enforce bool
// RealtimeReservedBytes is the part of Global.Bytes that only realtime
// tables may use, so queued backlog cannot consume the space realtime
// ingestion depends on. It applies in both enforced and observe-only
// modes, and only when Policy has realtime tables.
RealtimeReservedBytes int64
Policy *ingestpolicy.Policy
}
func ResolveAdmission ¶ added in v0.4.0
func ResolveAdmission(global Usage, maxBytes, maxSegments int64, disabled bool, overrides []TableLimit) (AdmissionOpts, error)
ResolveAdmission applies per-instance defaults. Zero means 10% of the global capacity; disabling defaults still permits explicit table overrides.
func ResolveAdmissionWithEnforcement ¶ added in v0.4.0
func ResolveAdmissionWithEnforcement(global Usage, maxBytes, maxSegments int64, disabled, enforce bool, overrides []TableLimit) (AdmissionOpts, error)
ResolveAdmissionWithEnforcement resolves budgets and selects whether exceeding a table budget rejects writes or is only reported.
type Destination ¶ added in v0.4.0
Destination identifies a normalized storage table, independently of its schema.
func DestinationFromFilename ¶ added in v0.4.0
func DestinationFromFilename(path string) (Destination, error)
func DestinationFromPrefix ¶ added in v0.4.0
func DestinationFromPrefix(prefix string) (Destination, error)
func (Destination) String ¶ added in v0.4.0
func (d Destination) String() string
type Index ¶
type Index struct {
// Admission owns live capacity accounting and is notified when a segment is
// removed so its budget is released.
Admission *Admission
// contains filtered or unexported fields
}
Index provides overview of all segments in a repository.
The index owns closed-segment bookkeeping only. Live capacity accounting is owned by Admission; the index notifies it when a segment is removed so the two never maintain parallel counters that can diverge.
func (*Index) Add ¶
func (i *Index) Add(s SegmentInfo)
Add adds a closed segment to the index and notifies subscribers.
func (*Index) Contains ¶ added in v0.4.0
Contains returns true if the index has a segment with path.
func (*Index) Get ¶
func (i *Index) Get(infos []SegmentInfo, prefix string) []SegmentInfo
Get returns all segments for a given prefix.
func (*Index) LargestCountPrefix ¶
LargestCountPrefix returns the prefix of the segment with the largest total count.
func (*Index) LargestSizePrefix ¶
LargestSizePrefix returns the prefix of the segment with the largest total size.
func (*Index) Oldest ¶ added in v0.4.0
func (i *Index) Oldest(n int) []SegmentInfo
Oldest returns up to n of the oldest segments in the index, oldest first.
func (*Index) OldestPrefix ¶
Oldest returns the prefix of the oldest segment.
func (*Index) OldestSegmentAge ¶ added in v0.2.0
OldestSegmentAge returns the age of the oldest segment in the index or 0 if the index is empty.
func (*Index) Prefixes ¶ added in v0.4.0
Prefixes appends all prefixes to dst in no particular order and returns it.
func (*Index) PrefixesByAge ¶
PrefixesByAge returns all prefixes sorted by oldest to newest.
func (*Index) PrefixesByCount ¶
PrefixesByCount returns all prefixes sorted by total count least to greatest. If there is a tie, the prefix that is lexicographically first is returned.
func (*Index) PrefixesBySize ¶
PrefixesBySize returns all prefixes sorted by total size least to greatest.
func (*Index) Remove ¶
func (i *Index) Remove(s SegmentInfo)
Remove removes a segment from the index and releases the capacity it held.
func (*Index) Subscribe ¶ added in v0.4.0
func (i *Index) Subscribe(fn func(SegmentInfo)) (unsubscribe func())
Subscribe registers fn to be called each time a closed segment is added to the index. fn is called synchronously by the goroutine that added the segment, after the index lock is released, so it must return quickly and must not block. Segments added before Subscribe are not replayed; callers should read the index after subscribing to discover existing segments. The returned func unsubscribes fn.
func (*Index) TotalPrefixes ¶
TotalPrefixes returns the total number of prefixes in the index.
func (*Index) TotalSegments ¶
TotalSegments returns the total number of segments in the index.
func (*Index) TotalSizeByPriority ¶ added in v0.4.0
func (i *Index) TotalSizeByPriority(p ingestpolicy.Priority) int64
TotalSizeByPriority returns the total size of segments in the index with the given ingestion priority.
type Iterator ¶
type Iterator interface {
Next() (bool, error)
Value() []byte
Close() error
// Verify ensures the Iterator can iterate over a continuous series of blocks without error.
Verify() (int, error)
Metadata() (SampleType, uint32)
}
func NewSegmentIterator ¶
func NewSegmentIterator(r io.ReadCloser) (Iterator, error)
type Option ¶
type Option func(*SegmentReader)
type Repository ¶
type Repository struct {
// contains filtered or unexported fields
}
Repository is a collection of WALs.
func NewRepository ¶
func NewRepository(opts RepositoryOpts) *Repository
func (*Repository) Admission ¶ added in v0.4.0
func (s *Repository) Admission() *Admission
func (*Repository) Close ¶
func (s *Repository) Close() error
func (*Repository) Count ¶
func (s *Repository) Count() int
func (*Repository) Index ¶
func (s *Repository) Index() *Index
func (*Repository) IsActiveSegment ¶ added in v0.4.0
func (s *Repository) IsActiveSegment(path string) bool
IsActiveSegment returns true if path is the active, not yet closed, segment of a WAL.
func (*Repository) Keys ¶
func (s *Repository) Keys() [][]byte
func (*Repository) MaxDiskUsage ¶ added in v0.4.0
func (s *Repository) MaxDiskUsage(p ingestpolicy.Priority) int64
MaxDiskUsage returns the total disk usage at which writes with the given priority are rejected. Queued writes are limited to MaxDiskUsage minus the realtime reservation so realtime writes can continue when queued data backs up. A value of 0 means no limit.
func (*Repository) PrefixesByAge ¶
func (s *Repository) PrefixesByAge() []string
func (*Repository) Remove ¶
func (s *Repository) Remove(key []byte) error
func (*Repository) RemoveSegment ¶
func (s *Repository) RemoveSegment(si SegmentInfo)
func (*Repository) Size ¶ added in v0.2.0
func (s *Repository) Size() int64
func (*Repository) WriteDebug ¶ added in v0.2.0
func (s *Repository) WriteDebug(w io.Writer) error
type RepositoryOpts ¶
type RepositoryOpts struct {
StorageDir string
SegmentMaxSize int64
SegmentMaxAge time.Duration
StartupOpenConcurrency int
MaxDiskUsage int64
MaxSegmentCount int
WALFlushInterval time.Duration
EnableWALFsync bool
// Policy assigns an ingestion priority to each WAL prefix. When nil, all WALs are queued.
Policy *ingestpolicy.Policy
// Realtime overrides the segment rotation settings for realtime WALs.
Realtime RotationPolicy
// UnownedSegmentMaxAge, when positive, replaces SegmentMaxAge for queued WALs of tables that Owned reports are
// owned by another node. Their segments are transferred to the owner rather than uploaded, so holding them here
// only delays them. Ownership is checked when a WAL is created and again by each rotation sweep.
UnownedSegmentMaxAge time.Duration
// Owned reports whether this node owns the table of a WAL prefix. When nil, every table is owned.
Owned func(prefix string) bool
// RealtimeReservedDiskBytes is disk space reserved for realtime WALs. Queued WALs reject writes once total disk
// usage reaches MaxDiskUsage minus this value. It only applies when Policy has realtime tables and is less than
// MaxDiskUsage.
RealtimeReservedDiskBytes int64
// Admission owns per-destination capacity accounting for this repository.
Admission *Admission
}
type Reservation ¶ added in v0.4.0
type Reservation struct {
// contains filtered or unexported fields
}
func (*Reservation) Finish ¶ added in v0.4.0
func (r *Reservation) Finish(bytes int64)
Finish commits the bytes actually written, or rolls back with zero. Segment reservations are committed only when a segment header was created.
func (*Reservation) Release ¶ added in v0.4.0
func (r *Reservation) Release()
Release rolls back an unfinished reservation and returns its allocation to the pool. The caller must not access the reservation after Release.
type RotationPolicy ¶ added in v0.4.0
type RotationPolicy struct {
SegmentMaxAge time.Duration
SegmentMaxSize int64
WALFlushInterval time.Duration
}
RotationPolicy overrides segment rotation settings. Zero values inherit the repository settings.
type SampleType ¶
type SampleType uint16
const ( UnknownSampleType SampleType = iota MetricSampleType TraceSampleType LogSampleType )
func SampleMetadata ¶
func SampleMetadata(b []byte) (t SampleType, count uint32)
type Segment ¶
type Segment interface {
Append(ctx context.Context, buf []byte) (int, error)
Write(ctx context.Context, buf []byte, opts ...WriteOptions) (int, error)
Bytes() ([]byte, error)
Close() error
ID() string
Size() int64
CreatedAt() time.Time
Reader() (io.ReadCloser, error)
Path() string
Iterator() (Iterator, error)
Info() SegmentInfo
Flush() error
// Repair truncates the last bytes in the segment if they are missing, corrupted or have extra data. This
// repairs any corrupted segment that may not have fully flushed to disk safely. The segment is truncated
// from the first block that is found to be corrupt.
Repair() error
}
func NewSegment ¶
func NewSegment(dir, prefix string, opts ...SegmentOption) (Segment, error)
type SegmentInfo ¶
type SegmentInfo struct {
Prefix string
Ulid string
Path string
Size int64
CreatedAt time.Time
// RawSize is the uncompressed size of the segment's values, as they are ingested, or zero when it is not known.
RawSize int64
// Priority is the ingestion priority of the segment's table.
Priority ingestpolicy.Priority
}
type SegmentMerger ¶
type SegmentMerger struct {
// contains filtered or unexported fields
}
func (*SegmentMerger) Close ¶
func (m *SegmentMerger) Close() error
type SegmentOption ¶
type SegmentOption func(s *segment)
func WithAdmission ¶ added in v0.4.0
func WithAdmission(a *Admission) SegmentOption
func WithFlushIntervale ¶
func WithFlushIntervale(d time.Duration) SegmentOption
func WithFsync ¶ added in v0.2.0
func WithFsync(b bool) SegmentOption
type SegmentReader ¶
type SegmentReader struct {
// contains filtered or unexported fields
}
func NewSegmentReader ¶
func NewSegmentReader(path string, opts ...Option) (*SegmentReader, error)
func (*SegmentReader) Close ¶
func (s *SegmentReader) Close() error
Close closes the segment's file. It returns nil if the reader already closed it at the end of the segment.
func (*SegmentReader) Path ¶ added in v0.2.0
func (s *SegmentReader) Path() string
func (*SegmentReader) SampleMetadata ¶
func (s *SegmentReader) SampleMetadata() (t SampleType, sampleCount uint32)
type SegmentTooLargeError ¶ added in v0.4.0
type SegmentTooLargeError struct {
Destination Destination
Size int64
Limit int64
}
SegmentTooLargeError reports a payload that cannot fit in a destination's budget even when that destination is completely empty. Unlike TableOverloadedError this is not relieved by draining, so the sender must split the batch or an operator must raise the limit.
func (*SegmentTooLargeError) Error ¶ added in v0.4.0
func (e *SegmentTooLargeError) Error() string
type TableBudget ¶ added in v0.4.0
type TableBudget struct {
// contains filtered or unexported fields
}
TableBudget owns the admission state for one destination. Writers resolve it once and reuse it, so the write path touches only this destination's lock and the process-wide counters. Budgets for different tables never contend.
func (*TableBudget) Reserve ¶ added in v0.4.0
func (t *TableBudget) Reserve(path string, bytes, segments int64) (*Reservation, error)
Reserve claims capacity for a pending write. The caller must call Finish with the bytes actually written and then Release.
type TableLimit ¶ added in v0.4.0
type TableLimit struct {
Database string `toml:"database" json:"database"`
Table string `toml:"table" json:"table"`
MaxBytes int64 `toml:"max-bytes" json:"max-bytes"`
MaxSegments int64 `toml:"max-segments" json:"max-segments"`
}
TableLimit overrides the default local backlog budget for a normalized table.
func ParseTableLimit ¶ added in v0.4.0
func ParseTableLimit(value string) (TableLimit, error)
type TableOverloadedError ¶ added in v0.4.0
type TableOverloadedError struct {
Destination Destination
Reason string
RetryAfter time.Duration
}
func (*TableOverloadedError) Error ¶ added in v0.4.0
func (e *TableOverloadedError) Error() string
type TableStatus ¶ added in v0.4.0
type TableStatus struct {
Usage
Limit Usage
Throttled bool
Enforced bool
WouldReject int64
RejectedRecords, RejectedBytes int64
OldestAge time.Duration
// ThrottledFor is how long this destination has been continuously over
// budget. A brief value is a burst; a sustained one means the destination
// either needs a larger budget or is genuinely producing more than this
// instance can drain.
ThrottledFor time.Duration
}
type Usage ¶ added in v0.4.0
Usage counts bytes and files, including reservations when used by admission.
type WAL ¶
type WAL struct {
// contains filtered or unexported fields
}
func (*WAL) Priority ¶ added in v0.4.0
func (w *WAL) Priority() ingestpolicy.Priority
Priority returns the ingestion priority of the WAL's table.
func (*WAL) SegmentMaxAge ¶ added in v0.4.1
SegmentMaxAge returns the age at which the WAL's segments rotate. 0 means they do not rotate by age.
func (*WAL) SetSegmentMaxAge ¶ added in v0.4.1
SetSegmentMaxAge changes the age at which the WAL's segments rotate, including the current segment. 0 stops rotating them by age.
type WALOpts ¶
type WALOpts struct {
StorageDir string
// WAL segment prefix
Prefix string
// SegmentMaxSize is the max size of a segment in bytes before it will be rotated and compressed.
SegmentMaxSize int64
// SegmentMaxAge is the max age of a segment before it will be rotated and compressed.
SegmentMaxAge time.Duration
// MaxDiskUsage is the max disk usage of WAL segments allowed before writes should be rejected.
MaxDiskUsage int64
// MaxSegmentCount is the max number of segments allowed before writes should be rejected.
MaxSegmentCount int
// Index is the index of the WAL segments.
Index *Index
// WALFlushInterval is the interval at which the WAL should be flushed.
WALFlushInterval time.Duration
// EnableWALFsync enables fsync of the segment after every flush.
EnableWALFsync bool
// Priority is the ingestion priority of the WAL's table.
Priority ingestpolicy.Priority
// Admission owns per-destination capacity accounting. When nil, table
// budgets are not applied to this WAL.
Admission *Admission
// contains filtered or unexported fields
}
type WriteOptions ¶
type WriteOptions func([]byte)
func WithSampleMetadata ¶
func WithSampleMetadata(t SampleType, count uint32) WriteOptions