Documentation
¶
Index ¶
- Constants
- Variables
- func ConvertExtensions(extensions any, v any) (any, error)
- func ReadMarker(ctx context.Context, logger log.Logger, bkt objstore.InstrumentedBucketReader, ...) error
- type DeletionMark
- type DeletionRequest
- type File
- type HashFunc
- type IndexStats
- type Marker
- type Matchers
- type Meta
- type NoCompactMark
- type NoCompactReason
- type NoDownsampleMark
- type NoDownsampleReason
- type ObjectHash
- type Rewrite
- type SourceType
- type Thanos
- type ThanosDownsample
Constants ¶
const ( // DeletionMarkFilename is the known json filename for optional file storing details about when block is marked for deletion. // If such file is present in block dir, it means the block is meant to be deleted after certain delay. DeletionMarkFilename = "deletion-mark.json" // NoCompactMarkFilename is the known json filename for optional file storing details about why block has to be excluded from compaction. // If such file is present in block dir, it means the block has to excluded from compaction (both vertical and horizontal) or rewrite (e.g deletions). NoCompactMarkFilename = "no-compact-mark.json" // NoDownsampleMarkFilename is the known json filenanme for optional file storing details about why block has to be excluded from downsampling. // If such file is present in block dir, it means the block has to be excluded from downsampling. NoDownsampleMarkFilename = "no-downsample-mark.json" // DeletionMarkVersion1 is the version of deletion-mark file supported by Thanos. DeletionMarkVersion1 = 1 // NoCompactMarkVersion1 is the version of no-compact-mark file supported by Thanos. NoCompactMarkVersion1 = 1 // NoDownsampleVersion1 is the version of no-downsample-mark file supported by Thanos. NoDownsampleMarkVersion1 = 1 )
const ( // ManualNoCompactReason is a custom reason of excluding from compaction that should be added when no-compact mark is added for unknown/user specified reason. ManualNoCompactReason NoCompactReason = "manual" // ManualNoDownsampleReason is a custom reason of excluding from downsample that should be added when no-downsample mark is added for unknown/user specified reason. ManualNoDownsampleReason NoDownsampleReason = "manual" // IndexSizeExceedingNoCompactReason is a reason of index being too big (for example exceeding 64GB limit: https://github.com/thanos-io/thanos/issues/1424) // This reason can be ignored when vertical block sharding will be implemented. IndexSizeExceedingNoCompactReason = "index-size-exceeding" // OutOfOrderChunksNoCompactReason is a reason of to no compact block with index contains out of order chunk so that the compaction is not blocked. OutOfOrderChunksNoCompactReason = "block-index-out-of-order-chunk" // DownsampleVerticalCompactionNoCompactReason is a reason to not compact overlapping downsampled blocks as it does not make sense e.g. how to vertically compact the average. DownsampleVerticalCompactionNoCompactReason = "downsample-vertical-compaction" )
const ( // MetaFilename is the known JSON filename for meta information. MetaFilename = "meta.json" // TSDBVersion1 is a enumeration of TSDB meta versions supported by Thanos. TSDBVersion1 = 1 // ThanosVersion1 is a enumeration of Thanos section of TSDB meta supported by Thanos. ThanosVersion1 = 1 // ParquetMigratedExtensionKey is the key used in block extensions to indicate // that the block has been migrated to parquet format and can be safely ignored // by store gateways. ParquetMigratedExtensionKey = "parquet_migrated" )
Variables ¶
var ( // ErrorMarkerNotFound is the error when marker file is not found. ErrorMarkerNotFound = errors.New("marker not found") // ErrorUnmarshalMarker is the error when unmarshalling marker JSON file. // This error can occur because marker has been partially uploaded to block storage // or the marker file is not a valid json file. ErrorUnmarshalMarker = errors.New("unmarshal marker JSON") )
Functions ¶
func ConvertExtensions ¶ added in v0.32.0
ConvertExtensions converts extensions with `any` type into specific type `v` that the caller expects.
Types ¶
type DeletionMark ¶ added in v0.12.0
type DeletionMark struct {
// ID of the tsdb block.
ID ulid.ULID `json:"id"`
// Version of the file.
Version int `json:"version"`
// Details is a human readable string giving details of reason.
Details string `json:"details,omitempty"`
// DeletionTime is a unix timestamp of when the block was marked to be deleted.
DeletionTime int64 `json:"deletion_time"`
}
DeletionMark stores block id and when block was marked for deletion.
type DeletionRequest ¶ added in v0.18.0
type DeletionRequest struct {
Matchers Matchers `json:"matchers" yaml:"matchers"`
Intervals tombstones.Intervals `json:"intervals,omitempty" yaml:"intervals,omitempty"`
RequestID string `json:"request_id,omitempty" yaml:"request_id,omitempty"`
}
type File ¶ added in v0.17.0
type File struct {
RelPath string `json:"rel_path"`
// SizeBytes is optional (e.g meta.json does not show size).
SizeBytes int64 `json:"size_bytes,omitempty"`
// Hash is an optional hash of this file. Used for potentially avoiding an extra download.
Hash *ObjectHash `json:"hash,omitempty"`
}
type IndexStats ¶ added in v0.32.0
type Meta ¶
Meta describes the a block's meta. It wraps the known TSDB meta structure and extends it by Thanos-specific fields.
func InjectThanos ¶
func InjectThanos(logger log.Logger, bdir string, meta Thanos, downsampledMeta *tsdb.BlockMeta) (*Meta, error)
InjectThanos sets Thanos meta to the block meta JSON and saves it to the disk. NOTE: It should be used after writing any block by any Thanos component, otherwise we will miss crucial metadata.
func Read ¶
func Read(rc io.ReadCloser) (_ *Meta, err error)
Read the block meta from the given reader.
func ReadFromDir ¶ added in v0.18.0
ReadFromDir reads the given meta from <dir>/meta.json.
type NoCompactMark ¶ added in v0.17.0
type NoCompactMark struct {
// ID of the tsdb block.
ID ulid.ULID `json:"id"`
// Version of the file.
Version int `json:"version"`
// Details is a human readable string giving details of reason.
Details string `json:"details,omitempty"`
// NoCompactTime is a unix timestamp of when the block was marked for no compact.
NoCompactTime int64 `json:"no_compact_time"`
Reason NoCompactReason `json:"reason"`
}
NoCompactMark marker stores reason of block being excluded from compaction if needed.
type NoCompactReason ¶ added in v0.17.0
type NoCompactReason string
NoCompactReason is a reason for a block to be excluded from compaction.
type NoDownsampleMark ¶ added in v0.30.0
type NoDownsampleMark struct {
// ID of the tsdb block.
ID ulid.ULID `json:"id"`
// Version of the file.
Version int `json:"version"`
// Details is a human readable string giving details of reason.
Details string `json:"details,omitempty"`
// NoDownsampleTime is a unix timestamp of when the block was marked for no downsample.
NoDownsampleTime int64 `json:"no_downsample_time"`
Reason NoDownsampleReason `json:"reason"`
}
NoDownsampleMark marker stores reason of block being excluded from downsample if needed.
type NoDownsampleReason ¶ added in v0.30.0
type NoDownsampleReason string
NoDownsampleReason is a reason for a block to be excluded from downsample.
type ObjectHash ¶ added in v0.19.0
ObjectHash stores the hash of an object in the object storage.
func CalculateHash ¶ added in v0.19.0
CalculateHash calculates the hash of the given type.
func (*ObjectHash) Equal ¶ added in v0.19.0
func (oh *ObjectHash) Equal(other *ObjectHash) bool
Equal returns true if two hashes are equal.
type Rewrite ¶ added in v0.18.0
type Rewrite struct {
// ULIDs of all source head blocks that went into the block.
Sources []ulid.ULID `json:"sources,omitempty"`
// Deletions if applied (in order).
DeletionsApplied []DeletionRequest `json:"deletions_applied,omitempty"`
// Relabels if applied.
RelabelsApplied []*relabel.Config `json:"relabels_applied,omitempty"`
}
type SourceType ¶
type SourceType string
const ( // TODO(bwplotka): Merge with pkg/component package. UnknownSource SourceType = "" SidecarSource SourceType = "sidecar" ReceiveSource SourceType = "receive" CompactorSource SourceType = "compactor" CompactorRepairSource SourceType = "compactor.repair" RulerSource SourceType = "ruler" BucketRepairSource SourceType = "bucket.repair" BucketRewriteSource SourceType = "bucket.rewrite" BucketUploadSource SourceType = "bucket.upload" TestSource SourceType = "test" )
type Thanos ¶
type Thanos struct {
// Version of Thanos meta file. If none specified, 1 is assumed (since first version did not have explicit version specified).
Version int `json:"version,omitempty"`
// Labels are the external labels identifying the producer as well as tenant.
// See https://thanos.io/tip/thanos/storage.md#external-labels for details.
Labels map[string]string `json:"labels"`
Downsample ThanosDownsample `json:"downsample"`
// Source is a real upload source of the block.
Source SourceType `json:"source"`
// List of segment files (in chunks directory), in sorted order. Optional.
// Deprecated. Use Files instead.
SegmentFiles []string `json:"segment_files,omitempty"`
// File is a sorted (by rel path) list of all files in block directory of this block known to TSDB.
// Sorted by relative path.
// Useful to avoid API call to get size of each file, as well as for debugging purposes.
// Optional, added in v0.17.0.
Files []File `json:"files,omitempty"`
// Rewrites is present when any rewrite (deletion, relabel etc) were applied to this block. Optional.
Rewrites []Rewrite `json:"rewrites,omitempty"`
// IndexStats contains stats info related to block index.
IndexStats IndexStats `json:"index_stats,omitempty"`
// Extensions are used for plugin any arbitrary additional information for block. Optional.
Extensions any `json:"extensions,omitempty"`
// UploadTime is used to track when the meta.json file was uploaded to the object storage
// without an extra Attributes call. Used for consistency filter.
UploadTime time.Time `json:"upload_time,omitempty"`
}
Thanos holds block meta information specific to Thanos.
func (*Thanos) GroupKey ¶ added in v0.25.0
GroupKey returns a unique identifier for the compaction group the block belongs to. It considers the downsampling resolution and the block's labels.
func (*Thanos) ParseExtensions ¶ added in v0.32.0
func (*Thanos) ResolutionString ¶ added in v0.32.0
ResolutionString returns a the block's resolution as a string.
type ThanosDownsample ¶
type ThanosDownsample struct {
Resolution int64 `json:"resolution"`
}