Help us shape the future of Go! Take the 2026 Go Developer Survey

metastore

package
v3.7.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 17, 2026 License: AGPL-3.0 Imports: 45 Imported by: 0

Documentation

Index

Constants

View Source 
const (

	// TocPrefix is the prefix under which ToC files are stored in the object storage.
	TocPrefix = "tocs/"
)

Variables

View Source 
var (
	ErrInvalidLengthMetastore = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowMetastore   = fmt.Errorf("proto: integer overflow")
)

Functions

This section is empty.

Types

type ArrowRecordBatchReader added in v3.7.0 

type ArrowRecordBatchReader interface {
	Open(ctx context.Context) error
	Read(ctx context.Context) (arrow.RecordBatch, error)
	Close()
}

type CollectSectionsRequest added in v3.7.0 

type CollectSectionsRequest struct {
	Reader ArrowRecordBatchReader
}

type CollectSectionsResponse added in v3.7.0 

type CollectSectionsResponse struct {
	SectionsResponse SectionsResponse
}

type Config added in v3.6.0 

type Config struct {
	IndexStoragePrefix string `yaml:"index_storage_prefix" experimental:"true"`
	PartitionRatio     int    `yaml:"partition_ratio" experimental:"true"`
}

Config is the configuration block for the metastore settings.

func (*Config) RegisterFlags added in v3.6.0 

func (c *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the flags for the metastore settings.

func (*Config) Validate added in v3.6.0 

func (c *Config) Validate() error

Validate validates the metastore settings.

type DataobjSectionDescriptor added in v3.6.0 

type DataobjSectionDescriptor struct {
	SectionKey

	StreamIDs []int64
	RowCount  int
	Size      int64
	Start     time.Time
	End       time.Time

	// Ambiguous predicates are predicates which are present in the stream's labels as well as the LogQL query, and are therefore ambiguous.
	AmbiguousPredicatesByStream map[int64][]string
}

DataobjSectionDescriptor is a descriptor for single section of a data object, containing some useful information about that section.

func NewSectionDescriptor added in v3.6.0 

func NewSectionDescriptor(pointer pointers.SectionPointer, ambiguousLabelNames []string) *DataobjSectionDescriptor

NewSectionDescriptor creates a new section descriptor with the given pointer and labels.

func (*DataobjSectionDescriptor) Merge added in v3.6.0 

func (d *DataobjSectionDescriptor) Merge(pointer pointers.SectionPointer, lbls []string)

Merge merges the given pointer and labels into an existing section's descriptor.

type GetIndexesRequest added in v3.7.0 

type GetIndexesRequest struct {
	Start time.Time
	End   time.Time
}

type GetIndexesResponse added in v3.7.0 

type GetIndexesResponse struct {
	TableOfContentsPaths []string
	IndexesPaths         []string
}

type IndexSectionsReaderRequest added in v3.7.0 

type IndexSectionsReaderRequest struct {
	IndexPath       string
	SectionsRequest SectionsRequest
	PrefetchBytes   int64
	BatchSize       int
}

type IndexSectionsReaderResponse added in v3.7.0 

type IndexSectionsReaderResponse struct {
	Reader ArrowRecordBatchReader
}

type Metastore 

type Metastore interface {
	// Sections returns a list of SectionDescriptors, including metadata (stream IDs, start & end times, bytes), for the given matchers & predicates between [start,end]
	Sections(ctx context.Context, req SectionsRequest) (SectionsResponse, error)

	GetIndexes(ctx context.Context, req GetIndexesRequest) (GetIndexesResponse, error)
	IndexSectionsReader(ctx context.Context, req IndexSectionsReaderRequest) (IndexSectionsReaderResponse, error)
	CollectSections(ctx context.Context, req CollectSectionsRequest) (CollectSectionsResponse, error)

	// Labels returns all possible labels from matching streams between [start,end]
	Labels(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error) // Used to get possible labels for a given stream

	// Values returns all possible values for the given label matchers between [start,end]
	Values(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error) // Used to get all values for a given set of label matchers
}

type ObjectMetastore 

type ObjectMetastore struct {
	// contains filtered or unexported fields
}

ObjectMetastore is a metastore that stores data objects in object storage.

func NewObjectMetastore 

func NewObjectMetastore(b objstore.Bucket, cfg Config, logger log.Logger, metrics *ObjectMetastoreMetrics) *ObjectMetastore

func (*ObjectMetastore) CollectSections added in v3.7.0 

func (m *ObjectMetastore) CollectSections(ctx context.Context, req CollectSectionsRequest) (CollectSectionsResponse, error)

func (*ObjectMetastore) DataObjects 

func (m *ObjectMetastore) DataObjects(ctx context.Context, start, end time.Time, _ ...*labels.Matcher) ([]string, error)

func (*ObjectMetastore) GetIndexes added in v3.7.0 

func (m *ObjectMetastore) GetIndexes(ctx context.Context, req GetIndexesRequest) (GetIndexesResponse, error)

func (*ObjectMetastore) IndexSectionsReader added in v3.7.0 

func (m *ObjectMetastore) IndexSectionsReader(ctx context.Context, req IndexSectionsReaderRequest) (IndexSectionsReaderResponse, error)

func (*ObjectMetastore) Labels 

func (m *ObjectMetastore) Labels(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error)

func (*ObjectMetastore) Sections added in v3.6.0 

func (m *ObjectMetastore) Sections(ctx context.Context, req SectionsRequest) (SectionsResponse, error)

func (*ObjectMetastore) Values 

func (m *ObjectMetastore) Values(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error)

type ObjectMetastoreMetrics added in v3.7.0 

type ObjectMetastoreMetrics struct {
	// contains filtered or unexported fields
}

func NewObjectMetastoreMetrics added in v3.7.0 

func NewObjectMetastoreMetrics(reg prometheus.Registerer) *ObjectMetastoreMetrics

type ObjectWrittenEvent added in v3.6.0 

type ObjectWrittenEvent struct {
	ObjectPath         string `protobuf:"bytes,2,opt,name=objectPath,proto3" json:"objectPath,omitempty"`
	WriteTime          string `protobuf:"bytes,3,opt,name=writeTime,proto3" json:"writeTime,omitempty"`
	EarliestRecordTime string `protobuf:"bytes,4,opt,name=earliestRecordTime,proto3" json:"earliestRecordTime,omitempty"`
}

ObjectWrittenEvent describes a new logs object being written to object storage.

func (*ObjectWrittenEvent) Descriptor added in v3.6.0 

func (*ObjectWrittenEvent) Descriptor() ([]byte, []int)

func (*ObjectWrittenEvent) Equal added in v3.6.0 

func (this *ObjectWrittenEvent) Equal(that interface{}) bool

func (*ObjectWrittenEvent) GetEarliestRecordTime added in v3.7.0 

func (m *ObjectWrittenEvent) GetEarliestRecordTime() string

func (*ObjectWrittenEvent) GetObjectPath added in v3.6.0 

func (m *ObjectWrittenEvent) GetObjectPath() string

func (*ObjectWrittenEvent) GetWriteTime added in v3.6.0 

func (m *ObjectWrittenEvent) GetWriteTime() string

func (*ObjectWrittenEvent) GoString added in v3.6.0 

func (this *ObjectWrittenEvent) GoString() string

func (*ObjectWrittenEvent) Marshal added in v3.6.0 

func (m *ObjectWrittenEvent) Marshal() (dAtA []byte, err error)

func (*ObjectWrittenEvent) MarshalTo added in v3.6.0 

func (m *ObjectWrittenEvent) MarshalTo(dAtA []byte) (int, error)

func (*ObjectWrittenEvent) MarshalToSizedBuffer added in v3.6.0 

func (m *ObjectWrittenEvent) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ObjectWrittenEvent) ProtoMessage added in v3.6.0 

func (*ObjectWrittenEvent) ProtoMessage()

func (*ObjectWrittenEvent) Reset added in v3.6.0 

func (m *ObjectWrittenEvent) Reset()

func (*ObjectWrittenEvent) Size added in v3.6.0 

func (m *ObjectWrittenEvent) Size() (n int)

func (*ObjectWrittenEvent) String added in v3.6.0 

func (this *ObjectWrittenEvent) String() string

func (*ObjectWrittenEvent) Unmarshal added in v3.6.0 

func (m *ObjectWrittenEvent) Unmarshal(dAtA []byte) error

func (*ObjectWrittenEvent) XXX_DiscardUnknown added in v3.6.0 

func (m *ObjectWrittenEvent) XXX_DiscardUnknown()

func (*ObjectWrittenEvent) XXX_Marshal added in v3.6.0 

func (m *ObjectWrittenEvent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ObjectWrittenEvent) XXX_Merge added in v3.6.0 

func (m *ObjectWrittenEvent) XXX_Merge(src proto.Message)

func (*ObjectWrittenEvent) XXX_Size added in v3.6.0 

func (m *ObjectWrittenEvent) XXX_Size() int

func (*ObjectWrittenEvent) XXX_Unmarshal added in v3.6.0 

func (m *ObjectWrittenEvent) XXX_Unmarshal(b []byte) error

type SectionKey added in v3.6.0 

type SectionKey struct {
	ObjectPath string
	SectionIdx int64
}

SectionKey is a unique identifier for a section of a data object.

type SectionsRequest added in v3.7.0 

type SectionsRequest struct {
	Start      time.Time
	End        time.Time
	Matchers   []*labels.Matcher
	Predicates []*labels.Matcher
}

type SectionsResponse added in v3.7.0 

type SectionsResponse struct {
	Sections []*DataobjSectionDescriptor
}

type TableOfContentsWriter added in v3.6.0 

type TableOfContentsWriter struct {
	// contains filtered or unexported fields
}

The TableOfContents (ToC) writer manages the metastore's Table of Contents files, which are a list of other data objects in storage for a particular time range. The Table of Contents files are used to look up other objects based on a time range, either index files or the log objects themselves. All entries are expected to have an applicable time window.

func NewTableOfContentsWriter added in v3.6.0 

func NewTableOfContentsWriter(bucket objstore.Bucket, logger log.Logger) *TableOfContentsWriter

NewTableOfContentsWriter creates a new Writer for adding entries to the metastore's Table of Contents files.

func (*TableOfContentsWriter) RegisterMetrics added in v3.6.0 

func (m *TableOfContentsWriter) RegisterMetrics(reg prometheus.Registerer) error

func (*TableOfContentsWriter) UnregisterMetrics added in v3.6.0 

func (m *TableOfContentsWriter) UnregisterMetrics(reg prometheus.Registerer)

func (*TableOfContentsWriter) WriteEntry added in v3.6.0 

func (m *TableOfContentsWriter) WriteEntry(ctx context.Context, dataobjPath string, tenantTimeRanges []multitenancy.TimeRange) error

WriteEntry adds the provided path to the Table of Contents file. The min/max timestamps are stored as metastore for the new entry can be accessed by time.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.