pkg

package module
v0.0.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 25, 2025 License: Apache-2.0 Imports: 15 Imported by: 7

Documentation

Index

Constants

View Source 
const DefaultMaxFrameSize = (4 << 20) - 1024

DefaultMaxFrameSize is the default maximum size of a frame. 4MiB, less 1KiB to ensure the frame fits in default gRPC message size, which is 4MiB.

View Source 
const DefaultMaxTotalDictSize = 4 << 20

DefaultMaxTotalDictSize is the default maximum of MaxTotalDictSize option.

View Source 
const EmptyBytes = Bytes("")
View Source 
const HdrContentSizeLimit = 1 << 20
View Source 
const (
	HdrFlagsCompressionMethod byte = 0b00000011
)
View Source 
const (
	HdrFormatVersion = 0
)
View Source 
const (
	HdrFormatVersionMask byte = 0x0F
)
View Source 
const HdrSignature = "STEF"
View Source 
const MultimapElemCountLimit = 1024

Variables

View Source 
var EndOfFrame = errors.New("end of frame")
View Source 
var ErrEndOfFrame = errors.New("end of frame")
View Source 
var ErrInvalidCompression = errors.New("invalid compression method")
View Source 
var ErrInvalidFormatVersion = errors.New("invalid format version in the FixedHeader")
View Source 
var ErrInvalidHeader = errors.New("invalid FixedHeader")
View Source 
var ErrInvalidHeaderSignature = errors.New("invalid FixedHeader signature")
View Source 
var ErrInvalidOneOfType = errors.New("invalid oneof type")
View Source 
var ErrInvalidRefNum = errors.New("invalid refNum")
View Source 
var ErrMultimap = errors.New("invalid multimap")
View Source 
var ErrMultimapCountLimit = errors.New("too many elements in the multimap")

Functions

func BoolCompare 

func BoolCompare(left, right bool) int

func BoolEqual 

func BoolEqual(left, right bool) bool

func BoolRandom added in v0.0.4 

func BoolRandom(random *rand.Rand) bool

func BytesCompare 

func BytesCompare(left, right Bytes) int

func BytesEqual 

func BytesEqual(left, right Bytes) bool

func EnsureLen 

func EnsureLen[T byte | int64 | float64 | any](data []T, dataLen int) []T

func Float64Compare 

func Float64Compare(left, right float64) int

func Float64Equal 

func Float64Equal(left, right float64) bool

func Float64Random added in v0.0.4 

func Float64Random(random *rand.Rand) float64

func Int64Compare 

func Int64Compare(left, right int64) int

func Int64Equal 

func Int64Equal(left, right int64) bool

func Int64Random added in v0.0.4 

func Int64Random(random *rand.Rand) int64

func StringCompare added in v0.0.7 

func StringCompare(left, right string) int

func StringEqual 

func StringEqual(left, right string) bool

func StringRandom added in v0.0.4 

func StringRandom(random *rand.Rand) string

func Uint64Compare 

func Uint64Compare(left, right uint64) int

func Uint64Equal 

func Uint64Equal(left, right uint64) bool

func Uint64Random added in v0.0.4 

func Uint64Random(random *rand.Rand) uint64

Types

type BaseReader 

type BaseReader struct {
	Source *bufio.Reader

	FixedHeader FixedHeader
	VarHeader   VarHeader
	Schema      *schema.WireSchema

	ReadBufs ReadBufs

	FrameDecoder     FrameDecoder
	FrameRecordCount uint64
	RecordCount      uint64
}

func (*BaseReader) Init 

func (r *BaseReader) Init(source *bufio.Reader) error

func (*BaseReader) NextFrame 

func (r *BaseReader) NextFrame() (FrameFlags, error)

func (*BaseReader) ReadFixedHeader 

func (r *BaseReader) ReadFixedHeader() error

func (*BaseReader) ReadVarHeader 

func (r *BaseReader) ReadVarHeader(ownSchema schema.WireSchema) error

type BitsReader 

type BitsReader struct {
	// contains filtered or unexported fields
}

BitsReader is a reader of bits.

func NewBitsReader 

func NewBitsReader() *BitsReader

func (*BitsReader) Consume 

func (b *BitsReader) Consume(nbits uint)

Consume advances the bit pointer by nbits bits. Consume must be preceded by PeekBits() call with at least the same value of nbits. Maximum allowed value for count is 56.

func (*BitsReader) IsEOF 

func (b *BitsReader) IsEOF() bool

func (*BitsReader) MapBytesFromMemBuf 

func (b *BitsReader) MapBytesFromMemBuf(src *BytesReader, byteSize int) error

func (*BitsReader) PeekBits 

func (b *BitsReader) PeekBits(nbits uint) uint64

PeekBits must ensure at least nbits bits become available, i.e. b.availBitCount >= nbits on return. If this means going past EOF then zero bits are appended at the end. Maximum allowed value for nbits is 56.

func (*BitsReader) ReadBit 

func (b *BitsReader) ReadBit() uint64

func (*BitsReader) ReadBits 

func (b *BitsReader) ReadBits(nbits uint) uint64

ReadBits reads bits. nbits should be in [0..64] range. Reading past EOF produces additional 0 bits on the least significant side. IsEOF condition will be set if ReadBits is called again after consuming all available bits.

func (*BitsReader) ReadUvarintCompact 

func (b *BitsReader) ReadUvarintCompact() uint64

ReadUvarintCompact reads a variable-bit encoded unsigned value. The range of supported values is [0..2^48-1].

func (*BitsReader) ReadVarintCompact 

func (b *BitsReader) ReadVarintCompact() int64

ReadVarintCompact reads a variable-bit encoded signed value. The range of supported values is [-2^47..2^47-1].

func (*BitsReader) Reset 

func (b *BitsReader) Reset(buf []byte)

type BitsWriter 

type BitsWriter struct {
	// contains filtered or unexported fields
}

BitsWriter is a stream of bits

func NewBitsWriter 

func NewBitsWriter(size int) BitsWriter

func (*BitsWriter) BitCount 

func (b *BitsWriter) BitCount() uint

func (*BitsWriter) Bytes 

func (b *BitsWriter) Bytes() []byte

func (*BitsWriter) Close 

func (b *BitsWriter) Close()

Close finalizes pending write operations. Must be called before Bytes(). If you want to reuse the BitsWriter after calling Close() then call Reset() before using this writer.

func (*BitsWriter) Reset 

func (b *BitsWriter) Reset()

func (*BitsWriter) WriteBit 

func (b *BitsWriter) WriteBit(bit uint)

func (*BitsWriter) WriteBits 

func (b *BitsWriter) WriteBits(val uint64, nbits uint)

func (*BitsWriter) WriteUvarintCompact 

func (b *BitsWriter) WriteUvarintCompact(val uint64) uint

WriteUvarintCompact writes a variable-bit encoded value. The range of supported values is [0..2^48-1]. Returns the number of bits written.

func (*BitsWriter) WriteVarintCompact 

func (b *BitsWriter) WriteVarintCompact(val int64) uint

WriteVarintCompact reads a variable-bit encoded signed value. The range of supported values is [-2^47..2^47-1]. Returns the number of bits written.

type ByteAndBlockReader 

type ByteAndBlockReader interface {
	ReadByte() (byte, error)
	Read(p []byte) (n int, err error)
}

type Bytes 

type Bytes string

Bytes is a sequence of immutable bytes.

func BytesRandom added in v0.0.4 

func BytesRandom(random *rand.Rand) Bytes

type BytesReader 

type BytesReader struct {
	// contains filtered or unexported fields
}

func (*BytesReader) MapBytesFromMemBuf 

func (r *BytesReader) MapBytesFromMemBuf(src *BytesReader, byteSize int) error

func (*BytesReader) ReadByte 

func (r *BytesReader) ReadByte() (byte, error)

func (*BytesReader) ReadBytesMapped 

func (r *BytesReader) ReadBytesMapped(byteSize int) ([]byte, error)

func (*BytesReader) ReadStringBytes 

func (r *BytesReader) ReadStringBytes(byteSize int) (string, error)

func (*BytesReader) ReadStringMapped added in v0.0.8 

func (r *BytesReader) ReadStringMapped(byteSize int) (string, error)

func (*BytesReader) ReadUvarint 

func (r *BytesReader) ReadUvarint() (value uint64, err error)

func (*BytesReader) ReadVarint 

func (r *BytesReader) ReadVarint() (value int64, err error)

func (*BytesReader) Reset 

func (r *BytesReader) Reset(buf []byte)

type BytesWriter 

type BytesWriter struct {
	// contains filtered or unexported fields
}

func NewBytesWriter 

func NewBytesWriter(cap int) BytesWriter

func (*BytesWriter) Bytes 

func (w *BytesWriter) Bytes() []byte

func (*BytesWriter) MapBytesToBitsReader 

func (w *BytesWriter) MapBytesToBitsReader(dest *BitsReader, byteSize int) error

func (*BytesWriter) Reset 

func (w *BytesWriter) Reset()

func (*BytesWriter) ResetAndReserve 

func (w *BytesWriter) ResetAndReserve(len int)

func (*BytesWriter) WriteByte 

func (w *BytesWriter) WriteByte(b byte)

func (*BytesWriter) WriteBytes 

func (w *BytesWriter) WriteBytes(bytes []byte)

func (*BytesWriter) WriteStringBytes 

func (w *BytesWriter) WriteStringBytes(val string)

func (*BytesWriter) WriteUvarint 

func (w *BytesWriter) WriteUvarint(value uint64)

func (*BytesWriter) WriteVarint 

func (w *BytesWriter) WriteVarint(x int64)

type ChunkWriter 

type ChunkWriter interface {
	// WriteChunk writes a chunk of bytes, first the header, then the content.
	WriteChunk(header []byte, content []byte) error
}

ChunkWriter allows to write a chunk: either a header or a frame of STEF byte stream.

type Compression 

type Compression int
const (
	CompressionNone Compression = 0
	CompressionZstd Compression = 1
	CompressionMask             = 0b11
)

type FixedHeader 

type FixedHeader struct {
	Compression Compression
}

type FrameDecoder 

type FrameDecoder struct {
	// contains filtered or unexported fields
}

func (*FrameDecoder) Init 

func (d *FrameDecoder) Init(src ByteAndBlockReader, compression Compression) error

func (*FrameDecoder) Next 

func (d *FrameDecoder) Next() (FrameFlags, error)

func (*FrameDecoder) Read 

func (d *FrameDecoder) Read(p []byte) (n int, err error)

func (*FrameDecoder) ReadByte 

func (d *FrameDecoder) ReadByte() (byte, error)

func (*FrameDecoder) RemainingSize 

func (d *FrameDecoder) RemainingSize() uint64

type FrameEncoder 

type FrameEncoder struct {
	// contains filtered or unexported fields
}

func (*FrameEncoder) CloseFrame 

func (e *FrameEncoder) CloseFrame() error

func (*FrameEncoder) Init 

func (e *FrameEncoder) Init(dest ChunkWriter, compr Compression) error

func (*FrameEncoder) OpenFrame 

func (e *FrameEncoder) OpenFrame(resetFlags FrameFlags)

func (*FrameEncoder) UncompressedSize 

func (e *FrameEncoder) UncompressedSize() int

func (*FrameEncoder) Write 

func (e *FrameEncoder) Write(p []byte) (n int, err error)

type FrameFlags 

type FrameFlags byte
const (
	// RestartDictionaries resets and restarts all dictionaries at frame beginning.
	RestartDictionaries FrameFlags = 1 << iota
	// RestartCompression resets and restarts the compression stream at frame beginning.
	RestartCompression
	// RestartCodecs resets the state of all encoders/decoders at frame beginning.
	RestartCodecs

	FrameFlagsMask = RestartDictionaries | RestartCompression | RestartCodecs
)

type MemChunkWriter 

type MemChunkWriter struct {
	// contains filtered or unexported fields
}

MemChunkWriter is a ChunkWriter that accumulates chunks in a memory buffer.

func (*MemChunkWriter) Bytes 

func (m *MemChunkWriter) Bytes() []byte

func (*MemChunkWriter) WriteChunk 

func (m *MemChunkWriter) WriteChunk(header []byte, content []byte) error

type ReadBufs 

type ReadBufs struct {
	Columns ReadColumnSet
	// contains filtered or unexported fields
}

func (*ReadBufs) ReadFrom 

func (s *ReadBufs) ReadFrom(buf ByteAndBlockReader) error

type ReadColumnSet 

type ReadColumnSet struct {
	// contains filtered or unexported fields
}

func (*ReadColumnSet) AddSubColumn 

func (s *ReadColumnSet) AddSubColumn() *ReadColumnSet

func (*ReadColumnSet) Column 

func (s *ReadColumnSet) Column() *ReadableColumn

func (*ReadColumnSet) PrintSchema 

func (s *ReadColumnSet) PrintSchema(indent int)

func (*ReadColumnSet) ReadDataFrom 

func (s *ReadColumnSet) ReadDataFrom(buf ByteAndBlockReader) error

func (*ReadColumnSet) ReadSizesFrom 

func (s *ReadColumnSet) ReadSizesFrom(buf *BitsReader) error

func (*ReadColumnSet) ResetData 

func (s *ReadColumnSet) ResetData()

func (*ReadColumnSet) SubColumnLen 

func (s *ReadColumnSet) SubColumnLen() int

type ReadOptions added in v0.0.4 

type ReadOptions struct {
	// TillEndOfFrame indicates that Read() operation must only succeed if
	// the record is available in the already fetched frame. This ensures
	// that the Reader will not attempt to fetch new frames from the underlying
	// source reader and guarantees that it will not block on I/O.
	// If Read() is attempted when there are no more record remaining in the
	// current frame then ErrEndOfFrame is returned by Read().
	// If TillEndOfFrame is false then Read() will fetch new frames from the
	// underlying reader as needed.
	TillEndOfFrame bool
}

type ReadableColumn 

type ReadableColumn struct {
	// contains filtered or unexported fields
}

func (*ReadableColumn) BorrowData added in v0.0.8 

func (c *ReadableColumn) BorrowData() []byte

BorrowData returns the data and sets the internal data to nil. This allows to avoid copying the data if the caller wants to take exclusive ownership.

func (*ReadableColumn) Data 

func (c *ReadableColumn) Data() []byte

type SizeLimiter 

type SizeLimiter struct {
	// contains filtered or unexported fields
}

SizeLimiter tracks the current (approximate) byte size of dictionaries and of of the frame against the specified size limit.

func (*SizeLimiter) AddDictElemSize 

func (d *SizeLimiter) AddDictElemSize(elemByteSize uint)

AddDictElemSize accounts for adding an element of specified bytes size to the dictionary.

func (*SizeLimiter) AddFrameBits 

func (d *SizeLimiter) AddFrameBits(bitCount uint)

AddFrameBits accounts for adding bytes to the frame buffer.

func (*SizeLimiter) AddFrameBytes 

func (d *SizeLimiter) AddFrameBytes(byteCount uint)

func (*SizeLimiter) DictLimitReached 

func (d *SizeLimiter) DictLimitReached() bool

DictLimitReached returns true if the accumulated added element sizes reaches the previously defined limit. If specified limit was 0 the limit is never reached and this function will always return false.

func (*SizeLimiter) FrameLimitReached 

func (d *SizeLimiter) FrameLimitReached() bool

FrameLimitReached returns true if the accumulated added byte sizes reaches the previously defined limit. If specified limit was 0 the limit is never reached and this function will always return false.

func (*SizeLimiter) Init 

func (d *SizeLimiter) Init(opts *WriterOptions)

Init prepares limiter operation. dictByteSizeLimit specifies the limit of size in bytes. 0 means no limit, so the size won't be tracked at all.

func (*SizeLimiter) ResetDict 

func (d *SizeLimiter) ResetDict()

ResetDict resets accumulated sizes to 0 and DictLimitReached indicator to false. Normally used in conjunction with resetting the dictionary itself.

func (*SizeLimiter) ResetFrameSize 

func (d *SizeLimiter) ResetFrameSize()

ResetFrameSize resets accumulated sizes to 0 and FrameLimitReached indicator to false. Normally used after restarting the frame.

type VarHeader 

type VarHeader struct {
	SchemaWireBytes []byte
	UserData        map[string]string
}

func (*VarHeader) Deserialize added in v0.0.2 

func (h *VarHeader) Deserialize(src *bytes.Buffer) error

func (*VarHeader) Serialize added in v0.0.2 

func (h *VarHeader) Serialize(dst *bytes.Buffer) error

type WrapChunkWriter 

type WrapChunkWriter struct {
	// contains filtered or unexported fields
}

WrapChunkWriter is a ChunkWriter that writes to a wrapped io.Writer.

func NewWrapChunkWriter 

func NewWrapChunkWriter(w io.Writer) *WrapChunkWriter

func (*WrapChunkWriter) WriteChunk 

func (m *WrapChunkWriter) WriteChunk(header []byte, content []byte) error

type WriteBufs 

type WriteBufs struct {
	Columns WriteColumnSet
	// contains filtered or unexported fields
}

func (*WriteBufs) WriteTo 

func (w *WriteBufs) WriteTo(buf io.Writer) error

type WriteColumnSet 

type WriteColumnSet struct {
	// contains filtered or unexported fields
}

func (*WriteColumnSet) AddSubColumn 

func (s *WriteColumnSet) AddSubColumn() *WriteColumnSet

func (*WriteColumnSet) At 

func (s *WriteColumnSet) At(i int) *WriteColumnSet

At returns the subcolumn at specified index.

func (*WriteColumnSet) PrintSchema 

func (s *WriteColumnSet) PrintSchema(indent int)

func (*WriteColumnSet) SetBits 

func (s *WriteColumnSet) SetBits(b *BitsWriter)

func (*WriteColumnSet) SetBytes 

func (s *WriteColumnSet) SetBytes(b *BytesWriter)

func (*WriteColumnSet) TotalCount 

func (s *WriteColumnSet) TotalCount() uint

func (*WriteColumnSet) WriteDataTo 

func (s *WriteColumnSet) WriteDataTo(buf io.Writer) error

func (*WriteColumnSet) WriteSizesTo 

func (s *WriteColumnSet) WriteSizesTo(buf *BitsWriter)

type WriterOptions 

type WriterOptions struct {
	// IncludeDescriptor indicates that the schema descriptor must be written to the file.
	IncludeDescriptor bool

	// Compression to use for frame content.
	// CompressionNone disables the compression.
	// CompressionZstd uses zstd compression for frame content.
	Compression Compression

	// The maximum size of a frame in bytes (uncompressed size if compression is used).
	// If the content exceeds this size, the frame will be closed and a new frame
	// will be started. If unspecified DefaultMaxFrameSize will be used.
	//
	// Blocks never cross frame boundaries and full blocks are written
	// before a frame is closed, so the frame size may exceed this limit by the
	// size of the largest block.
	//
	// Note that the implementation will do its best to honor this value but it may be occasionally
	// exceeded. It is not guaranteed that the frames are always smaller than this size.
	MaxUncompressedFrameByteSize uint

	// When a frame is restarted these flags define additional behavior.
	//
	// RestartDictionaries - the dictionaries will be cleared. All new frames will
	//   start with new dictionaries. Can be used to limit the size of the
	//   dictionaries that the recipients must keep in memory. Note that the
	//   frames always restart dictionaries regardless of MaxTotalDictSize setting.
	//
	// RestartCompression - the compression stream is started anew.
	//   All new frames will start with a new compressor state.
	//   Can be used to make the frames within a file skipable. Each new
	//   frame's compression streams starts with a new state of encoder.
	//   If this flag is unspecified the state of the compression encoder
	//   carries over through the frames, which makes impossible to skip
	//   frames and start decompressing from the next frame.
	//   This flag has effect only if Compression!=CompressionNone.
	//
	// RestartEncoders - the encoder's state will be cleared. All new frames will
	//   start with initial state of encoders.
	//
	// A combination of RestartDictionaries|RestartCompression|RestartEncoders flags
	// ensures that a frame is readable and decodable on its own, without the need
	// to read any preceding frames.
	FrameRestartFlags FrameFlags

	// MaxTotalDictSize is the maximum total byte size of all dictionaries.
	//
	// Default is DefaultMaxTotalDictSize.
	//
	// The Writer will compute the total size of all dictionaries it creates
	// during encoding process. When this limit is reached the Writer will
	// reset the dictionaries and will start a new Frame with RestartDictionaries
	// frame flag set.
	// Note that when the limit is reached the dictionaries will be reset regardless
	// of the value in RestartDictionaries bit in FrameRestartFlags.
	//
	// The Writer's total byte size calculation is approximate.
	// It is expected that the real memory usage by dictionaries may somewhat
	// exceed MaxTotalDictSize before the Writer detects that the limit is reached.
	MaxTotalDictSize uint

	// Schema describes the desired wire schema to write the data in.
	// The schema must be compatible with Writer's native schema otherwise
	// an error will be returned when attempting to create the Writer.
	// In nil the Writer will write in its native schema.
	Schema *schema.WireSchema

	// UserData is optional user-defined data that will be stored in the header.
	UserData map[string]string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.