Documentation
¶
Index ¶
- Variables
- func BuildS3SSECustomerHeaders(b64Key string) (algorithm, key, keyMD5 string, err error)
- func ConvertLocalPathToRemote(diskName, localPath string) (string, error)
- func CopyObject(ctx context.Context, diskName string, srcSize int64, ...) (int64, error)
- func CopyObjectStreaming(ctx context.Context, srcStorage storage.RemoteStorage, ...) error
- func DeleteFile(ctx context.Context, diskName, remotePath string) error
- func GetFileReader(ctx context.Context, diskName, remotePath string) (io.ReadCloser, error)
- func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
- func PutFile(ctx context.Context, diskName, remotePath string, content []byte) error
- func ReadBoolText(scanner *bufio.Scanner) (bool, error)
- func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) ([]byte, error)
- func ReadInt64Text(scanner *bufio.Scanner) (int64, error)
- func ReadIntText(scanner *bufio.Scanner) (int, error)
- func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
- func WriteMetadataToFile(metadata *Metadata, path string) error
- type Metadata
- type MetadataVersion
- type Object
- type ObjectStorageConnection
- type ObjectStorageCredentials
- type PlainDiskLayout
- type PlainFile
- type StorageObject
Constants ¶
This section is empty.
Variables ¶
var DisksConnections = xsync.NewMapOf[*ObjectStorageConnection]()
var DisksCredentials = xsync.NewMapOf[ObjectStorageCredentials]()
var InitCredentialsAndConnectionsMutex sync.Mutex
var S3VirtualHostBucketRE = regexp.MustCompile(`((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss-data-acc|oss|eos)([.\-][a-z0-9\-.:]+))`)
S3VirtualHostBucketRE https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access
var SystemDisks = xsync.NewMapOf[clickhouse.Disk]()
Functions ¶
func BuildS3SSECustomerHeaders ¶ added in v2.7.0
BuildS3SSECustomerHeaders derives the full SSE-C header triplet (algorithm, key, key MD5) from a ClickHouse <server_side_encryption_customer_key_base64> value. ClickHouse only supports AES256 for SSE-C and stores the customer key base64-encoded; AWS S3 requires the MD5 of the raw key (also base64-encoded) in addition to the key and algorithm on every HeadObject/GetObject request against an SSE-C encrypted object, otherwise the server returns 400. See https://github.com/Altinity/clickhouse-backup/issues/1374
func CopyObject ¶
func CopyObjectStreaming ¶ added in v2.6.0
func CopyObjectStreaming(ctx context.Context, srcStorage storage.RemoteStorage, dstStorage storage.RemoteStorage, srcKey, dstKey string, limiter *bwlimit.Limiter) error
func GetFileReader ¶
func InitCredentialsAndConnections ¶
func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName string) error
func ReadFileContent ¶
func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string) ([]byte, error)
func WriteFileContent ¶
func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string, content []byte) error
func WriteMetadataToFile ¶
Types ¶
type Metadata ¶
type Metadata struct {
Version MetadataVersion
StorageObjectCount int
TotalSize int64
StorageObjects []StorageObject
RefCount int
ReadOnly bool
InlineData string
Path string
}
func ReadMetadataFromFile ¶
func ReadMetadataFromReader ¶
func ReadMetadataFromReader(metadataFile io.ReadCloser, path string) (*Metadata, error)
type MetadataVersion ¶
type MetadataVersion uint32
const ( VersionAbsolutePaths MetadataVersion = 1 VersionRelativePath MetadataVersion = 2 VersionReadOnlyFlag MetadataVersion = 3 VersionInlineData MetadataVersion = 4 VersionFullObjectKey MetadataVersion = 5 )
type ObjectStorageConnection ¶
type ObjectStorageConnection struct {
Type string
// MetadataType - `local` for VFS disks with local metadata files, `plain` / `plain_rewritable` for disks
// whose object keys are derived from logical paths (no local metadata files)
MetadataType string
S3 *storage.S3
AzureBlob *storage.AzureBlob
MetadataPath string
}
func (*ObjectStorageConnection) GetRemoteBucket ¶
func (c *ObjectStorageConnection) GetRemoteBucket() string
func (*ObjectStorageConnection) GetRemoteObjectDiskPath ¶ added in v2.6.0
func (c *ObjectStorageConnection) GetRemoteObjectDiskPath() string
func (*ObjectStorageConnection) GetRemotePath ¶
func (c *ObjectStorageConnection) GetRemotePath() string
func (*ObjectStorageConnection) GetRemoteStorage ¶
func (c *ObjectStorageConnection) GetRemoteStorage() storage.RemoteStorage
type ObjectStorageCredentials ¶
type ObjectStorageCredentials struct {
Type string // aws, gcs, azblob
EndPoint string
S3AccessKey string
S3SecretKey string
S3AssumeRole string
S3Region string
S3StorageClass string
S3SSECustomerKey string // <server_side_encryption_customer_key_base64>
S3SSEKMSKeyId string // <server_side_encryption_kms_key_id>
S3SSEKMSEncryptionContext string // <server_side_encryption_kms_encryption_context>
AzureAccountName string
AzureAccountKey string
AzureContainerName string
}
type PlainDiskLayout ¶ added in v2.8.1
type PlainDiskLayout struct {
// contains filtered or unexported fields
}
PlainDiskLayout resolves logical directory paths to remote object key prefixes for object storage disks with metadata_type=plain (key == logical path) and plain_rewritable (flat random tokens + __meta/<token>/prefix.path mapping)
func NewPlainDiskLayout ¶ added in v2.8.1
func NewPlainDiskLayout(ctx context.Context, diskName string) (*PlainDiskLayout, error)
NewPlainDiskLayout builds the layout for diskName, for plain_rewritable disks it lists <disk_root>/__meta/ and reads every prefix.path object (the same way ClickHouse builds its in-memory path map at startup)
func (*PlainDiskLayout) EnsureDir ¶ added in v2.8.1
EnsureDir returns the remote key prefix (relative to disk root) for logicalDir, for plain_rewritable it creates the __meta/<token>/prefix.path objects for the directory and every missing ancestor (ClickHouse builds its in-memory path map from prefix.path objects only, each directory level must have its own entry)
func (*PlainDiskLayout) ListTree ¶ added in v2.8.1
func (l *PlainDiskLayout) ListTree(ctx context.Context, logicalBase string) (map[string]PlainFile, error)
ListTree returns all data files under logicalBase (recursively): logical path relative to logicalBase -> remote object key (relative to disk root) and size
type PlainFile ¶ added in v2.8.1
type PlainFile struct {
// RemoteKey - object key relative to the disk root
RemoteKey string
Size int64
}
PlainFile describes one data object of a plain/plain_rewritable disk