 Documentation
      ¶
      Documentation
      ¶
    
    
  
    
  
    Index ¶
- Constants
- func AllUsers(_ string) (bool, error)
- func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig MetadataCacheConfig, ...) (objstore.Bucket, error)
- func HashBlockID(id ulid.ULID) uint32
- func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error)
- func TenantDeletionMarkExists(ctx context.Context, bkt objstore.BucketReader, userID string) (bool, error)
- func WriteTenantDeletionMark(ctx context.Context, bkt objstore.Bucket, userID string, ...) error
- type BlocksStorageConfig
- type BucketIndexConfig
- type BucketStoreConfig
- type CacheBackend
- type ChunksCacheConfig
- type DurationList
- type InMemoryIndexCacheConfig
- type IndexCacheConfig
- type MemcachedClientConfig
- type MetadataCacheConfig
- type TSDBConfig
- type TenantDeletionMark
- type UsersScanner
Constants ¶
const ( // TenantIDExternalLabel is the external label containing the tenant ID, // set when shipping blocks to the storage. TenantIDExternalLabel = "__org_id__" // IngesterIDExternalLabel is the external label containing the ingester ID, // set when shipping blocks to the storage. IngesterIDExternalLabel = "__ingester_id__" // ShardIDExternalLabel is the external label containing the shard ID // and can be used to shard blocks. ShardIDExternalLabel = "__shard_id__" // How often are open TSDBs checked for being idle and closed. DefaultCloseIdleTSDBInterval = 5 * time.Minute // How often to check for tenant deletion mark. DeletionMarkCheckInterval = 1 * time.Hour // Default minimum bucket size (bytes) of the chunk pool. ChunkPoolDefaultMinBucketSize = store.EstimatedMaxChunkSize // Default maximum bucket size (bytes) of the chunk pool. ChunkPoolDefaultMaxBucketSize = 50e6 )
const ( // IndexCacheBackendInMemory is the value for the in-memory index cache backend. IndexCacheBackendInMemory = "inmemory" // IndexCacheBackendMemcached is the value for the memcached index cache backend. IndexCacheBackendMemcached = "memcached" // IndexCacheBackendDefault is the value for the default index cache backend. IndexCacheBackendDefault = IndexCacheBackendInMemory )
const (
	CacheBackendMemcached = "memcached"
)
    const TenantDeletionMarkPath = "markers/tenant-deletion-mark.json"
    Relative to user-specific prefix.
Variables ¶
This section is empty.
Functions ¶
func AllUsers ¶
AllUsers returns true to each call and should be used whenever the UsersScanner should not filter out any user due to sharding.
func CreateCachingBucket ¶
func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig MetadataCacheConfig, bkt objstore.Bucket, logger log.Logger, reg prometheus.Registerer) (objstore.Bucket, error)
func HashBlockID ¶
HashBlockID returns a 32-bit hash of the block ID useful for ring-based sharding.
func NewIndexCache ¶
func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error)
NewIndexCache creates a new index cache based on the input configuration.
func TenantDeletionMarkExists ¶
func TenantDeletionMarkExists(ctx context.Context, bkt objstore.BucketReader, userID string) (bool, error)
Checks for deletion mark for tenant. Errors other than "object not found" are returned.
func WriteTenantDeletionMark ¶
func WriteTenantDeletionMark(ctx context.Context, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider, mark *TenantDeletionMark) error
Uploads deletion mark to the tenant location in the bucket.
Types ¶
type BlocksStorageConfig ¶
type BlocksStorageConfig struct {
	Bucket      bucket.Config     `yaml:",inline"`
	BucketStore BucketStoreConfig `` /* 141-byte string literal not displayed */
	TSDB        TSDBConfig        `yaml:"tsdb"`
}
    BlocksStorageConfig holds the config information for the blocks storage.
func (*BlocksStorageConfig) RegisterFlags ¶
func (cfg *BlocksStorageConfig) RegisterFlags(f *flag.FlagSet)
RegisterFlags registers the TSDB flags
func (*BlocksStorageConfig) Validate ¶
func (cfg *BlocksStorageConfig) Validate() error
Validate the config.
type BucketIndexConfig ¶
type BucketIndexConfig struct {
	Enabled               bool          `yaml:"enabled"`
	UpdateOnErrorInterval time.Duration `yaml:"update_on_error_interval"`
	IdleTimeout           time.Duration `yaml:"idle_timeout"`
	MaxStalePeriod        time.Duration `yaml:"max_stale_period"`
}
    func (*BucketIndexConfig) RegisterFlagsWithPrefix ¶
func (cfg *BucketIndexConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)
type BucketStoreConfig ¶
type BucketStoreConfig struct {
	SyncDir                  string              `yaml:"sync_dir"`
	SyncInterval             time.Duration       `yaml:"sync_interval"`
	MaxConcurrent            int                 `yaml:"max_concurrent"`
	TenantSyncConcurrency    int                 `yaml:"tenant_sync_concurrency"`
	BlockSyncConcurrency     int                 `yaml:"block_sync_concurrency"`
	MetaSyncConcurrency      int                 `yaml:"meta_sync_concurrency"`
	ConsistencyDelay         time.Duration       `yaml:"consistency_delay"`
	IndexCache               IndexCacheConfig    `yaml:"index_cache"`
	ChunksCache              ChunksCacheConfig   `yaml:"chunks_cache"`
	MetadataCache            MetadataCacheConfig `yaml:"metadata_cache"`
	IgnoreDeletionMarksDelay time.Duration       `yaml:"ignore_deletion_mark_delay"`
	BucketIndex              BucketIndexConfig   `yaml:"bucket_index"`
	// Chunk pool.
	MaxChunkPoolBytes           uint64 `yaml:"max_chunk_pool_bytes"`
	ChunkPoolMinBucketSizeBytes int    `yaml:"chunk_pool_min_bucket_size_bytes" doc:"hidden"`
	ChunkPoolMaxBucketSizeBytes int    `yaml:"chunk_pool_max_bucket_size_bytes" doc:"hidden"`
	// Controls whether index-header lazy loading is enabled.
	IndexHeaderLazyLoadingEnabled     bool          `yaml:"index_header_lazy_loading_enabled"`
	IndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout"`
	// Controls the partitioner, used to aggregate multiple GET object API requests.
	// The config option is hidden until experimental.
	PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" doc:"hidden"`
	// Controls what is the ratio of postings offsets store will hold in memory.
	// Larger value will keep less offsets, which will increase CPU cycles needed for query touching those postings.
	// It's meant for setups that want low baseline memory pressure and where less traffic is expected.
	// On the contrary, smaller value will increase baseline memory usage, but improve latency slightly.
	// 1 will keep all in memory. Default value is the same as in Prometheus which gives a good balance.
	PostingOffsetsInMemSampling int `yaml:"postings_offsets_in_mem_sampling" doc:"hidden"`
}
    BucketStoreConfig holds the config information for Bucket Stores used by the querier and store-gateway.
func (*BucketStoreConfig) RegisterFlags ¶
func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet)
RegisterFlags registers the BucketStore flags
func (*BucketStoreConfig) Validate ¶
func (cfg *BucketStoreConfig) Validate() error
Validate the config.
type CacheBackend ¶
type CacheBackend struct {
	Backend   string                `yaml:"backend"`
	Memcached MemcachedClientConfig `yaml:"memcached"`
}
    type ChunksCacheConfig ¶
type ChunksCacheConfig struct {
	CacheBackend `yaml:",inline"`
	SubrangeSize        int64         `yaml:"subrange_size"`
	MaxGetRangeRequests int           `yaml:"max_get_range_requests"`
	AttributesTTL       time.Duration `yaml:"attributes_ttl"`
	SubrangeTTL         time.Duration `yaml:"subrange_ttl"`
}
    func (*ChunksCacheConfig) RegisterFlagsWithPrefix ¶
func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)
func (*ChunksCacheConfig) Validate ¶
func (cfg *ChunksCacheConfig) Validate() error
type DurationList ¶
DurationList is the block ranges for a tsdb
func (*DurationList) Set ¶
func (d *DurationList) Set(s string) error
Set implements the flag.Value interface
func (*DurationList) String ¶
func (d *DurationList) String() string
String implements the flag.Value interface
func (*DurationList) ToMilliseconds ¶
func (d *DurationList) ToMilliseconds() []int64
ToMilliseconds returns the duration list in milliseconds
type InMemoryIndexCacheConfig ¶
type InMemoryIndexCacheConfig struct {
	MaxSizeBytes uint64 `yaml:"max_size_bytes"`
}
    func (*InMemoryIndexCacheConfig) RegisterFlagsWithPrefix ¶
func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)
type IndexCacheConfig ¶
type IndexCacheConfig struct {
	Backend   string                   `yaml:"backend"`
	InMemory  InMemoryIndexCacheConfig `yaml:"inmemory"`
	Memcached MemcachedClientConfig    `yaml:"memcached"`
}
    func (*IndexCacheConfig) RegisterFlags ¶
func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet)
func (*IndexCacheConfig) RegisterFlagsWithPrefix ¶
func (cfg *IndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)
func (*IndexCacheConfig) Validate ¶
func (cfg *IndexCacheConfig) Validate() error
Validate the config.
type MemcachedClientConfig ¶
type MemcachedClientConfig struct {
	Addresses              string        `yaml:"addresses"`
	Timeout                time.Duration `yaml:"timeout"`
	MaxIdleConnections     int           `yaml:"max_idle_connections"`
	MaxAsyncConcurrency    int           `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize     int           `yaml:"max_async_buffer_size"`
	MaxGetMultiConcurrency int           `yaml:"max_get_multi_concurrency"`
	MaxGetMultiBatchSize   int           `yaml:"max_get_multi_batch_size"`
	MaxItemSize            int           `yaml:"max_item_size"`
	AutoDiscovery          bool          `yaml:"auto_discovery"`
}
    func (*MemcachedClientConfig) GetAddresses ¶
func (cfg *MemcachedClientConfig) GetAddresses() []string
func (*MemcachedClientConfig) RegisterFlagsWithPrefix ¶
func (cfg *MemcachedClientConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)
func (MemcachedClientConfig) ToMemcachedClientConfig ¶
func (cfg MemcachedClientConfig) ToMemcachedClientConfig() cacheutil.MemcachedClientConfig
func (*MemcachedClientConfig) Validate ¶
func (cfg *MemcachedClientConfig) Validate() error
Validate the config.
type MetadataCacheConfig ¶
type MetadataCacheConfig struct {
	CacheBackend `yaml:",inline"`
	TenantsListTTL          time.Duration `yaml:"tenants_list_ttl"`
	TenantBlocksListTTL     time.Duration `yaml:"tenant_blocks_list_ttl"`
	ChunksListTTL           time.Duration `yaml:"chunks_list_ttl"`
	MetafileExistsTTL       time.Duration `yaml:"metafile_exists_ttl"`
	MetafileDoesntExistTTL  time.Duration `yaml:"metafile_doesnt_exist_ttl"`
	MetafileContentTTL      time.Duration `yaml:"metafile_content_ttl"`
	MetafileMaxSize         int           `yaml:"metafile_max_size_bytes"`
	MetafileAttributesTTL   time.Duration `yaml:"metafile_attributes_ttl"`
	BlockIndexAttributesTTL time.Duration `yaml:"block_index_attributes_ttl"`
	BucketIndexContentTTL   time.Duration `yaml:"bucket_index_content_ttl"`
	BucketIndexMaxSize      int           `yaml:"bucket_index_max_size_bytes"`
}
    func (*MetadataCacheConfig) RegisterFlagsWithPrefix ¶
func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)
func (*MetadataCacheConfig) Validate ¶
func (cfg *MetadataCacheConfig) Validate() error
type TSDBConfig ¶
type TSDBConfig struct {
	Dir                       string        `yaml:"dir"`
	BlockRanges               DurationList  `yaml:"block_ranges_period"`
	Retention                 time.Duration `yaml:"retention_period"`
	ShipInterval              time.Duration `yaml:"ship_interval"`
	ShipConcurrency           int           `yaml:"ship_concurrency"`
	HeadCompactionInterval    time.Duration `yaml:"head_compaction_interval"`
	HeadCompactionConcurrency int           `yaml:"head_compaction_concurrency"`
	HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout"`
	HeadChunksWriteBufferSize int           `yaml:"head_chunks_write_buffer_size_bytes"`
	StripeSize                int           `yaml:"stripe_size"`
	WALCompressionEnabled     bool          `yaml:"wal_compression_enabled"`
	WALSegmentSizeBytes       int           `yaml:"wal_segment_size_bytes"`
	FlushBlocksOnShutdown     bool          `yaml:"flush_blocks_on_shutdown"`
	CloseIdleTSDBTimeout      time.Duration `yaml:"close_idle_tsdb_timeout"`
	// MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup.
	MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`
	// If true, user TSDBs are not closed on shutdown. Only for testing.
	// If false (default), user TSDBs are closed to make sure all resources are released and closed properly.
	KeepUserTSDBOpenOnShutdown bool `yaml:"-"`
	// How often to check for idle TSDBs for closing. DefaultCloseIdleTSDBInterval is not suitable for testing, so tests can override.
	CloseIdleTSDBInterval time.Duration `yaml:"-"`
	// Positive value enables experiemental support for exemplars. 0 or less to disable.
	MaxExemplars int `yaml:"max_exemplars"`
}
    TSDBConfig holds the config for TSDB opened in the ingesters.
func (*TSDBConfig) BlocksDir ¶
func (cfg *TSDBConfig) BlocksDir(userID string) string
BlocksDir returns the directory path where TSDB blocks and wal should be stored by the ingester
func (*TSDBConfig) IsBlocksShippingEnabled ¶
func (cfg *TSDBConfig) IsBlocksShippingEnabled() bool
IsShippingEnabled returns whether blocks shipping is enabled.
func (*TSDBConfig) RegisterFlags ¶
func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet)
RegisterFlags registers the TSDBConfig flags.
type TenantDeletionMark ¶
type TenantDeletionMark struct {
	// Unix timestamp when deletion marker was created.
	DeletionTime int64 `json:"deletion_time"`
	// Unix timestamp when cleanup was finished.
	FinishedTime int64 `json:"finished_time,omitempty"`
}
    func NewTenantDeletionMark ¶
func NewTenantDeletionMark(deletionTime time.Time) *TenantDeletionMark
func ReadTenantDeletionMark ¶
func ReadTenantDeletionMark(ctx context.Context, bkt objstore.BucketReader, userID string) (*TenantDeletionMark, error)
Returns tenant deletion mark for given user, if it exists. If it doesn't exist, returns nil mark, and no error.
type UsersScanner ¶
type UsersScanner struct {
	// contains filtered or unexported fields
}
    func NewUsersScanner ¶
func (*UsersScanner) ScanUsers ¶
func (s *UsersScanner) ScanUsers(ctx context.Context) (users, markedForDeletion []string, err error)
ScanUsers returns a fresh list of users found in the storage, that are not marked for deletion, and list of users marked for deletion.
If sharding is enabled, returned lists contains only the users owned by this instance.