vparquet

package
v0.0.0-...-2e42471 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 13, 2025 License: AGPL-3.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SearchPrevious = -1
	SearchNext     = -2
	NotFound       = -3

	SnapshotIDColumnName = "ID"
)
View Source 
const (
	Id = "id"

	LabelServiceName = "service.name"
	LabelCluster     = "cluster"
	LabelNamespace   = "namespace"
	LabelPod         = "pod"
	LabelContainer   = "container"

	LabelK8sClusterName   = "k8s.cluster.name"
	LabelK8sNamespaceName = "k8s.namespace.name"
	LabelK8sPodName       = "k8s.pod.name"
	LabelK8sContainerName = "k8s.container.name"
)

Label names for conversion b/n Proto <> Parquet

View Source 
const (
	DefinitionLevelSnapshot      = 0
	DefinitionLevelResourceAttrs = 1
	DefinitionLevelSnapshotAttrs = 1

	FieldResourceAttrKey       = "rs.Attrs.Key"
	FieldResourceAttrVal       = "rs.Attrs.Value"
	FieldResourceAttrValInt    = "rs.Attrs.ValueInt"
	FieldResourceAttrValDouble = "rs.Attrs.ValueDouble"
	FieldResourceAttrValBool   = "rs.Attrs.ValueBool"

	FieldAttrKey       = "attr.Key"
	FieldAttrVal       = "attr.Value"
	FieldAttrValInt    = "attr.ValueInt"
	FieldAttrValDouble = "attr.ValueDouble"
	FieldAttrValBool   = "attr.ValueBool"
)

These definition levels match the schema below

View Source 
const (
	DataFileName = "data.parquet"
)
View Source 
const (
	LabelDuration = "duration"
)

These are reserved search parameters

View Source 
const VersionString = "vParquet"

Variables

This section is empty.

Functions

func CopyBlock 

func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from backend.Reader, to backend.Writer) error

func CreateBlock 

func CreateBlock(ctx context.Context, cfg *common.BlockConfig, meta *backend.BlockMeta, i common.Iterator, r backend.Reader, to backend.Writer) (*backend.BlockMeta, error)

Types

type Attribute 

type Attribute struct {
	Key string `parquet:",snappy,dict"`

	// This is a bad design that leads to millions of null values. How can we fix this?
	Value       *string  `parquet:",dict,snappy,optional"  json:",omitempty"`
	ValueInt    *int64   `parquet:",snappy,optional"  json:",omitempty"`
	ValueDouble *float64 `parquet:",snappy,optional"  json:",omitempty"`
	ValueBool   *bool    `parquet:",snappy,optional"  json:",omitempty"`
	ValueKVList string   `parquet:",snappy,optional"  json:",omitempty"`
	ValueArray  string   `parquet:",snappy,optional"  json:",omitempty"`
}

type BackendReaderAt 

type BackendReaderAt struct {
	TotalBytesRead atomic.Uint64
	// contains filtered or unexported fields
}

BackendReaderAt is used to track backend requests and present a io.ReaderAt interface backed by a backend.Reader

func NewBackendReaderAt 

func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, blockID uuid.UUID, tenantID string) *BackendReaderAt

func (*BackendReaderAt) ReadAt 

func (b *BackendReaderAt) ReadAt(p []byte, off int64) (int, error)

func (*BackendReaderAt) ReadAtWithCache 

func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64) (int, error)

type Compactor 

type Compactor struct {
	// contains filtered or unexported fields
}

func NewCompactor 

func NewCompactor(opts common.CompactionOptions) *Compactor

func (*Compactor) Compact 

func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error)

type Encoding 

type Encoding struct{}

func (Encoding) CopyBlock 

func (v Encoding) CopyBlock(ctx context.Context, meta *backend.BlockMeta, from backend.Reader, to backend.Writer) error

func (Encoding) CreateBlock 

func (v Encoding) CreateBlock(ctx context.Context, cfg *common.BlockConfig, meta *backend.BlockMeta, i common.Iterator, r backend.Reader, to backend.Writer) (*backend.BlockMeta, error)

func (Encoding) CreateWALBlock 

func (v Encoding) CreateWALBlock(id uuid.UUID, tenantID string, filepath string, e backend.Encoding, dataEncoding string, ingestionSlack time.Duration) (common.WALBlock, error)

CreateWALBlock creates a new appendable block

func (Encoding) MigrateBlock 

func (v Encoding) MigrateBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from backend.Reader, to backend.Writer) error

func (Encoding) NewCompactor 

func (v Encoding) NewCompactor(opts common.CompactionOptions) common.Compactor

func (Encoding) OpenBlock 

func (v Encoding) OpenBlock(meta *backend.BlockMeta, r backend.Reader) (common.BackendBlock, error)

func (Encoding) OpenWALBlock 

func (v Encoding) OpenWALBlock(filename string, path string, ingestionSlack time.Duration, additionalStartSlack time.Duration) (common.WALBlock, error, error)

OpenWALBlock opens an existing appendable block

func (Encoding) OwnsWALBlock 

func (v Encoding) OwnsWALBlock(entry fs.DirEntry) bool

func (Encoding) Version 

func (v Encoding) Version() string

type LabelExpression 

type LabelExpression struct {
	Key        string `parquest:",snappy,dict"`
	Expression string `parquest:",snappy,dict"`
	Static     string `parquest:",snappy,dict"`
}

type MetricDefinition 

type MetricDefinition struct {
	Name       string            `parquest:",snappy,dict"`
	Labels     []LabelExpression `parquest:""`
	MetricType string            `parquest:",snappy,dict"`
	Expression *string           `parquest:",snappy,dict"`
	Namespace  *string           `parquest:",snappy,dict"`
	Help       *string           `parquest:",snappy,dict"`
	Unit       *string           `parquest:",snappy,dict"`
}

type MultiBlockIterator 

type MultiBlockIterator[T iterable] struct {
	// contains filtered or unexported fields
}

func (*MultiBlockIterator[T]) Close 

func (m *MultiBlockIterator[T]) Close()

func (*MultiBlockIterator[T]) Next 

func (m *MultiBlockIterator[T]) Next(ctx context.Context) (common.ID, T, error)

type RawIterator 

type RawIterator interface {
	Next(context.Context) (common.ID, parquet.Row, error)
	Close()
}

type Resource 

type Resource struct {
	Attrs []Attribute

	// Known attributes
	ServiceName      string  `parquet:",snappy,dict" json:",omitempty"`
	Cluster          *string `parquet:",snappy,optional,dict" json:",omitempty"`
	Namespace        *string `parquet:",snappy,optional,dict" json:",omitempty"`
	Pod              *string `parquet:",snappy,optional,dict" json:",omitempty"`
	Container        *string `parquet:",snappy,optional,dict" json:",omitempty"`
	K8sClusterName   *string `parquet:",snappy,optional,dict" json:",omitempty"`
	K8sNamespaceName *string `parquet:",snappy,optional,dict" json:",omitempty"`
	K8sPodName       *string `parquet:",snappy,optional,dict" json:",omitempty"`
	K8sContainerName *string `parquet:",snappy,optional,dict" json:",omitempty"`

	Test string `parquet:",snappy,dict,optional" json:",omitempty"` // Always empty for testing
}

type Snapshot 

type Snapshot struct {
	ID            []byte              `parquet:""`
	IDText        string              `parquet:",snappy"`
	Tracepoint    TracePointConfig    `parquet:"tp"`
	VarLookup     map[string]Variable `parquet:""`
	TsNanos       uint64              `parquet:",delta"`
	Frames        []StackFrame        `parquet:""`
	Watches       []WatchResult       `parquet:""`
	Attributes    []Attribute         `parquet:"attr"`
	DurationNanos uint64              `parquet:",delta"`
	Resource      Resource            `parquet:"rs"`
	LogMsg        *string             `parquet:",snappy,optional"`
}

type SnapshotIterator 

type SnapshotIterator interface {
	NextSnapshot(context.Context) (common.ID, *Snapshot, error)
	Close()
}

type StackFrame 

type StackFrame struct {
	FileName               string       `parquet:",snappy,dict"`
	MethodName             string       `parquet:",snappy,dict"`
	LineNumber             uint32       `parquet:",delta"`
	ClassName              *string      `parquet:",snappy,optional,dict"`
	IsAsync                bool         `parquet:""`
	ColumnNumber           *uint32      `parquet:",snappy,optional"`
	TranspiledFileName     *string      `parquet:",snappy,optional,dict"`
	TranspiledLineNumber   *uint32      `parquet:",snappy,optional"`
	TranspiledColumnNumber *uint32      `parquet:",snappy,optional"`
	Variables              []VariableID `parquet:""`
	AppFrame               bool         `parquet:""`
	NativeFrame            bool         `parquet:""`
	ShortPath              *string      `parquet:",snappy,dict,optional"`
}

type TracePointConfig 

type TracePointConfig struct {
	ID         string             `parquet:",snappy,dict"`
	Path       string             `parquet:",snappy,dict"`
	LineNumber uint32             `parquet:",delta"`
	Args       map[string]string  `parquet:""`
	Watches    []string           `parquet:""`
	Metrics    []MetricDefinition `parquet:""`
}

type Variable 

type Variable struct {
	Type      string       `parquet:",snappy,dict"`
	Value     string       `parquet:",snappy,dict"`
	Hash      string       `parquet:",snappy,dict"`
	Children  []VariableID `parquet:""`
	Truncated bool         `parquet:""`
}

type VariableID 

type VariableID struct {
	ID           string   `parquet:",snappy,dict"`
	Name         string   `parquet:",snappy,dict"`
	OriginalName *string  `parquet:",snappy,optional"`
	Modifiers    []string `parquet:""`
}

type WatchResult 

type WatchResult struct {
	Expression  string      `parquet:",snappy"`
	GoodResult  *VariableID `parquet:""`
	ErrorResult *string     `parquet:",snappy"`
	Source      string      `parquet:",snappy"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.