xdistikv

package module
v0.0.0-...-7840015 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 23, 2023 License: MIT Imports: 26 Imported by: 0

README

xdis-tikv

This library is used to implement the mapping between resp cmd(like redis commands) and tikv+pd with client-go

cmd

Documentation

Index

Constants

View Source
const (
	// max allowed databases
	MaxDatabases int = 128

	// max key size
	MaxKeySize int = 1024

	// max hash field size
	MaxHashFieldSize int = 1024

	// max zset member size
	MaxZSetMemberSize int = 1024

	// max set member size
	MaxSetMemberSize int = 1024

	// max value size
	MaxValueSize int = 1024 * 1024 * 1024

	// default scan count
	DefaultScanCount int = 10
)

For different const size configuration

View Source
const (
	StringName = "STRING"
	ListName   = "LIST"
	HashName   = "HASH"
	SetName    = "SET"
	ZSetName   = "ZSET"
	BitmapName = "BITMAP"
)

For different type name

View Source
const (
	NoneType byte = iota
	StringType

	HashType
	HSizeType

	ListType
	LMetaType

	SetType
	SSizeType

	ZSetType
	ZSizeType
	ZScoreType

	ExpTimeType
	ExpMetaType

	BitmapType
)

for backend store key type notice: Please add new type in order

View Source
const (
	UnionType byte = 51
	DiffType  byte = 52
	InterType byte = 53
)

For set op

View Source
const (
	MinScore     int64 = math.MinInt64 + 1
	MaxScore     int64 = math.MaxInt64
	InvalidScore int64 = math.MinInt64

	AggregateSum = "sum"
	AggregateMin = "min"
	AggregateMax = "max"
)

For zset op

View Source
const (
	BitAND = "and"
	BitOR  = "or"
	BitXOR = "xor"
	BitNot = "not"
)

For bit operation

View Source
const (
	LeaderPreKey = 255
	GCPreKey     = 256
)

For checker job operation key

View Source
const (
	RegisterStoragerName = "xdis-tikv"
)
View Source
const (
	ScanOnceNums = 1024
)

for scan

Variables

View Source
var (
	ErrCmdNotSupport = errors.New("ERR cmd not supported")

	ErrIntNumber   = errors.New("ERR invalid integer")
	ErrKeySize     = errors.New("ERR invalid key size")
	ErrValueSize   = errors.New("ERR invalid value size")
	ErrExpireValue = errors.New("ERR invalid expire value")

	ErrStringKey = errors.New("ERR invalid encode string key")

	ErrLMetaKey  = errors.New("ERR invalid lmeta key")
	ErrListKey   = errors.New("ERR invalid list key")
	ErrListSeq   = errors.New("ERR invalid list sequence, overflow")
	ErrListIndex = errors.New("ERR invalid list index")

	ErrHashKey       = errors.New("ERR invalid hash key")
	ErrHashIntVal    = errors.New("ERR hash value is not an integer")
	ErrHSizeKey      = errors.New("ERR invalid hsize key")
	ErrHashFieldSize = errors.New("ERR invalid hash field size")

	ErrSetKey        = errors.New("ERR invalid set key")
	ErrSSizeKey      = errors.New("ERR invalid ssize key")
	ErrSetMemberSize = errors.New("ERR invalid set member size")

	ErrZSizeKey         = errors.New("ERR invalid zsize key")
	ErrZSetKey          = errors.New("ERR invalid zset key")
	ErrZScoreKey        = errors.New("ERR invalid zscore key")
	ErrScoreOverflow    = errors.New("ERR zset score overflow")
	ErrInvalidAggregate = errors.New("ERR invalid aggregate")
	ErrInvalidWeightNum = errors.New("ERR invalid weight number")
	ErrInvalidSrcKeyNum = errors.New("ERR invalid src key number")
	ErrZSetMemberSize   = errors.New("ERR invalid zset member size")

	ErrExpMetaKey = errors.New("ERR invalid expire meta key")
	ErrExpTimeKey = errors.New("ERR invalid expire time key")

	ErrDataType = errors.New("ERR error data type")
	ErrMetaKey  = errors.New("ERR error meta key")

	ErrBitmapKey = errors.New("ERR invalid encode bitmap key")

	// For different common errors
	ErrScoreMiss = errors.New("ERR zset score miss")

	// for job err
	ErrLeaderValSize = errors.New("ERR invalid leader checker job val size")

	ErrTTLNoExp = errors.New("ERR TTL no exp to del")
)
View Source
var TypeName = map[byte]string{
	StringType:  "string",
	HashType:    "hash",
	HSizeType:   "hsize",
	ListType:    "list",
	LMetaType:   "lmeta",
	SetType:     "set",
	SSizeType:   "ssize",
	ZSetType:    "zset",
	ZSizeType:   "zsize",
	ZScoreType:  "zscore",
	ExpTimeType: "exptime",
	ExpMetaType: "expmeta",
	BitmapType:  "bitmap",
}

TypeName is the map of type -> name

Functions

func Int64

func Int64(v []byte, err error) (int64, error)

Int64 gets 64 integer with the little endian format.

func MaxInt32

func MaxInt32(a int32, b int32) int32

func MinInt32

func MinInt32(a int32, b int32) int32

func PutInt64

func PutInt64(v int64) []byte

PutInt64 puts the 64 integer.

func PutUInt64

func PutUInt64(v uint64) []byte

PutUInt64 puts the u64 integer.

func PutUint16

func PutUint16(v uint16) []byte

PutUint16 puts the u16 integer.

func Uint64

func Uint64(v []byte, err error) (uint64, error)

Uint64 gets unsigned 64 integer with the little endian format.

Types

type DB

type DB struct {
	// contains filtered or unexported fields
}

DB core sturct impl like redis string, list, hash, set, zset, bitmap struct store db op

func NewDB

func NewDB(store *Storager, idx int) *DB

func (*DB) Close

func (m *DB) Close() (err error)

func (*DB) DBBitmap

func (m *DB) DBBitmap() driver.IBitmapCmd

func (*DB) DBHash

func (m *DB) DBHash() driver.IHashCmd

func (*DB) DBList

func (m *DB) DBList() driver.IListCmd

func (*DB) DBSet

func (m *DB) DBSet() driver.ISetCmd

func (*DB) DBString

func (m *DB) DBString() driver.IStringCmd

func (*DB) DBZSet

func (m *DB) DBZSet() driver.IZsetCmd

func (*DB) FlushDB

func (db *DB) FlushDB(ctx context.Context) (drop int64, err error)

FlushDB flushes the data in select db.

func (*DB) Index

func (db *DB) Index() int

Index gets the index of database.

func (*DB) IndexVarBuf

func (db *DB) IndexVarBuf() []byte

IndexVarBuf gets the index varint buf of database.

func (*DB) SetIndex

func (db *DB) SetIndex(index int)

SetIndex set the index of database.

type DBBitmap

type DBBitmap struct {
	*DB
}

func NewDBBitmap

func NewDBBitmap(db *DB) *DBBitmap

func (*DBBitmap) BitCount

func (db *DBBitmap) BitCount(ctx context.Context, key []byte, start int, end int) (int64, error)

func (*DBBitmap) BitOP

func (db *DBBitmap) BitOP(ctx context.Context, op string, destKey []byte, srcKeys ...[]byte) (int64, error)

func (*DBBitmap) BitPos

func (db *DBBitmap) BitPos(ctx context.Context, key []byte, on int, start int, end int) (int64, error)

func (*DBBitmap) GetBit

func (db *DBBitmap) GetBit(ctx context.Context, key []byte, offset int) (int64, error)

func (*DBBitmap) SetBit

func (db *DBBitmap) SetBit(ctx context.Context, key []byte, offset int, on int) (int64, error)

type DBHash

type DBHash struct {
	*DB
}

func NewDBHash

func NewDBHash(db *DB) *DBHash

func (*DBHash) Del

func (db *DBHash) Del(ctx context.Context, keys ...[]byte) (int64, error)

func (*DBHash) Exists

func (db *DBHash) Exists(ctx context.Context, key []byte) (int64, error)

func (*DBHash) Expire

func (db *DBHash) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

func (*DBHash) ExpireAt

func (db *DBHash) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

func (*DBHash) HDel

func (db *DBHash) HDel(ctx context.Context, key []byte, args ...[]byte) (int64, error)

func (*DBHash) HGet

func (db *DBHash) HGet(ctx context.Context, key []byte, field []byte) ([]byte, error)

func (*DBHash) HGetAll

func (db *DBHash) HGetAll(ctx context.Context, key []byte) ([]driver.FVPair, error)

func (*DBHash) HIncrBy

func (db *DBHash) HIncrBy(ctx context.Context, key []byte, field []byte, delta int64) (int64, error)

func (*DBHash) HKeys

func (db *DBHash) HKeys(ctx context.Context, key []byte) ([][]byte, error)

func (*DBHash) HLen

func (db *DBHash) HLen(ctx context.Context, key []byte) (int64, error)

func (*DBHash) HMget

func (db *DBHash) HMget(ctx context.Context, key []byte, args ...[]byte) ([][]byte, error)

func (*DBHash) HMset

func (db *DBHash) HMset(ctx context.Context, key []byte, args ...driver.FVPair) error

func (*DBHash) HSet

func (db *DBHash) HSet(ctx context.Context, key []byte, field []byte, value []byte) (int64, error)

func (*DBHash) HValues

func (db *DBHash) HValues(ctx context.Context, key []byte) ([][]byte, error)

func (*DBHash) Persist

func (db *DBHash) Persist(ctx context.Context, key []byte) (int64, error)

func (*DBHash) TTL

func (db *DBHash) TTL(ctx context.Context, key []byte) (int64, error)

type DBList

type DBList struct {
	*DB
}

func NewDBList

func NewDBList(db *DB) *DBList

func (*DBList) BLPop

func (db *DBList) BLPop(ctx context.Context, keys [][]byte, timeout time.Duration) ([]interface{}, error)

todo: v2 list block pop, use PD(etcd) watch to do it

func (*DBList) BRPop

func (db *DBList) BRPop(ctx context.Context, keys [][]byte, timeout time.Duration) ([]interface{}, error)

func (*DBList) Del

func (db *DBList) Del(ctx context.Context, keys ...[]byte) (int64, error)

func (*DBList) Exists

func (db *DBList) Exists(ctx context.Context, key []byte) (int64, error)

func (*DBList) Expire

func (db *DBList) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

func (*DBList) ExpireAt

func (db *DBList) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

func (*DBList) LIndex

func (db *DBList) LIndex(ctx context.Context, key []byte, index int32) ([]byte, error)

func (*DBList) LLen

func (db *DBList) LLen(ctx context.Context, key []byte) (int64, error)

func (*DBList) LPop

func (db *DBList) LPop(ctx context.Context, key []byte) ([]byte, error)

func (*DBList) LPush

func (db *DBList) LPush(ctx context.Context, key []byte, args ...[]byte) (int64, error)

func (*DBList) LRange

func (db *DBList) LRange(ctx context.Context, key []byte, start int32, stop int32) ([][]byte, error)

func (*DBList) LSet

func (db *DBList) LSet(ctx context.Context, key []byte, index int32, value []byte) error

func (*DBList) LTrim

func (db *DBList) LTrim(ctx context.Context, key []byte, start, stop int64) error

func (*DBList) LTrimBack

func (db *DBList) LTrimBack(ctx context.Context, key []byte, trimSize int32) (int32, error)

func (*DBList) LTrimFront

func (db *DBList) LTrimFront(ctx context.Context, key []byte, trimSize int32) (int32, error)

func (*DBList) Persist

func (db *DBList) Persist(ctx context.Context, key []byte) (int64, error)

func (*DBList) RPop

func (db *DBList) RPop(ctx context.Context, key []byte) ([]byte, error)

func (*DBList) RPush

func (db *DBList) RPush(ctx context.Context, key []byte, args ...[]byte) (int64, error)

func (*DBList) TTL

func (db *DBList) TTL(ctx context.Context, key []byte) (int64, error)

type DBSet

type DBSet struct {
	*DB
}

func NewDBSet

func NewDBSet(db *DB) *DBSet

func (*DBSet) Del

func (db *DBSet) Del(ctx context.Context, keys ...[]byte) (int64, error)

func (*DBSet) Exists

func (db *DBSet) Exists(ctx context.Context, key []byte) (int64, error)

func (*DBSet) Expire

func (db *DBSet) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

func (*DBSet) ExpireAt

func (db *DBSet) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

func (*DBSet) Persist

func (db *DBSet) Persist(ctx context.Context, key []byte) (int64, error)

func (*DBSet) SAdd

func (db *DBSet) SAdd(ctx context.Context, key []byte, args ...[]byte) (int64, error)

func (*DBSet) SCard

func (db *DBSet) SCard(ctx context.Context, key []byte) (int64, error)

func (*DBSet) SDiff

func (db *DBSet) SDiff(ctx context.Context, keys ...[]byte) ([][]byte, error)

func (*DBSet) SDiffStore

func (db *DBSet) SDiffStore(ctx context.Context, dstKey []byte, keys ...[]byte) (int64, error)

func (*DBSet) SInter

func (db *DBSet) SInter(ctx context.Context, keys ...[]byte) ([][]byte, error)

func (*DBSet) SInterStore

func (db *DBSet) SInterStore(ctx context.Context, dstKey []byte, keys ...[]byte) (int64, error)

func (*DBSet) SIsMember

func (db *DBSet) SIsMember(ctx context.Context, key []byte, member []byte) (int64, error)

func (*DBSet) SMembers

func (db *DBSet) SMembers(ctx context.Context, key []byte) ([][]byte, error)

func (*DBSet) SRem

func (db *DBSet) SRem(ctx context.Context, key []byte, args ...[]byte) (int64, error)

func (*DBSet) SUnion

func (db *DBSet) SUnion(ctx context.Context, keys ...[]byte) ([][]byte, error)

func (*DBSet) SUnionStore

func (db *DBSet) SUnionStore(ctx context.Context, dstKey []byte, keys ...[]byte) (int64, error)

func (*DBSet) TTL

func (db *DBSet) TTL(ctx context.Context, key []byte) (int64, error)

type DBString

type DBString struct {
	*DB
}

func NewDBString

func NewDBString(db *DB) *DBString

func (*DBString) Append

func (db *DBString) Append(ctx context.Context, key []byte, value []byte) (int64, error)

func (*DBString) Decr

func (db *DBString) Decr(ctx context.Context, key []byte) (int64, error)

func (*DBString) DecrBy

func (db *DBString) DecrBy(ctx context.Context, key []byte, decrement int64) (int64, error)

func (*DBString) Del

func (db *DBString) Del(ctx context.Context, keys ...[]byte) (int64, error)

Del must atomic txn del

func (*DBString) Exists

func (db *DBString) Exists(ctx context.Context, key []byte) (int64, error)

func (*DBString) Expire

func (db *DBString) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

func (*DBString) ExpireAt

func (db *DBString) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

func (*DBString) Get

func (db *DBString) Get(ctx context.Context, key []byte) (val []byte, err error)

func (*DBString) GetRange

func (db *DBString) GetRange(ctx context.Context, key []byte, start int, end int) ([]byte, error)

func (*DBString) GetSet

func (db *DBString) GetSet(ctx context.Context, key []byte, value []byte) ([]byte, error)

func (*DBString) GetSlice

func (db *DBString) GetSlice(ctx context.Context, key []byte) (openkvdriver.ISlice, error)

func (*DBString) Incr

func (db *DBString) Incr(ctx context.Context, key []byte) (int64, error)

func (*DBString) IncrBy

func (db *DBString) IncrBy(ctx context.Context, key []byte, increment int64) (int64, error)

func (*DBString) MGet

func (db *DBString) MGet(ctx context.Context, keys ...[]byte) ([][]byte, error)

func (*DBString) MSet

func (db *DBString) MSet(ctx context.Context, args ...driver.KVPair) error

func (*DBString) Persist

func (db *DBString) Persist(ctx context.Context, key []byte) (int64, error)

func (*DBString) Set

func (db *DBString) Set(ctx context.Context, key []byte, value []byte) error

func (*DBString) SetEX

func (db *DBString) SetEX(ctx context.Context, key []byte, duration int64, value []byte) (err error)

func (*DBString) SetNX

func (db *DBString) SetNX(ctx context.Context, key []byte, value []byte) (n int64, err error)

func (*DBString) SetNXEX

func (db *DBString) SetNXEX(ctx context.Context, key []byte, duration int64, value []byte) (int64, error)

func (*DBString) SetRange

func (db *DBString) SetRange(ctx context.Context, key []byte, offset int, value []byte) (int64, error)

func (*DBString) SetXXEX

func (db *DBString) SetXXEX(ctx context.Context, key []byte, duration int64, value []byte) (int64, error)

func (*DBString) StrLen

func (db *DBString) StrLen(ctx context.Context, key []byte) (int64, error)

func (*DBString) TTL

func (db *DBString) TTL(ctx context.Context, key []byte) (int64, error)

type DBZSet

type DBZSet struct {
	*DB
}

func NewDBZSet

func NewDBZSet(db *DB) *DBZSet

func (*DBZSet) Del

func (db *DBZSet) Del(ctx context.Context, keys ...[]byte) (int64, error)

func (*DBZSet) Exists

func (db *DBZSet) Exists(ctx context.Context, key []byte) (int64, error)

func (*DBZSet) Expire

func (db *DBZSet) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

func (*DBZSet) ExpireAt

func (db *DBZSet) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

func (*DBZSet) Persist

func (db *DBZSet) Persist(ctx context.Context, key []byte) (int64, error)

func (*DBZSet) TTL

func (db *DBZSet) TTL(ctx context.Context, key []byte) (int64, error)

func (*DBZSet) ZAdd

func (db *DBZSet) ZAdd(ctx context.Context, key []byte, args ...driver.ScorePair) (int64, error)

func (*DBZSet) ZCard

func (db *DBZSet) ZCard(ctx context.Context, key []byte) (int64, error)

func (*DBZSet) ZCount

func (db *DBZSet) ZCount(ctx context.Context, key []byte, min int64, max int64) (int64, error)

func (*DBZSet) ZIncrBy

func (db *DBZSet) ZIncrBy(ctx context.Context, key []byte, delta int64, member []byte) (int64, error)

func (*DBZSet) ZInterStore

func (db *DBZSet) ZInterStore(ctx context.Context, destKey []byte, srcKeys [][]byte, weights []int64, aggregate []byte) (int64, error)

ZInterStore intersects the zsets and stores to dest zset.

func (*DBZSet) ZLexCount

func (db *DBZSet) ZLexCount(ctx context.Context, key []byte, min []byte, max []byte, rangeType driver.RangeType) (int64, error)

ZLexCount gets the count of zset lexicographically.

func (*DBZSet) ZRangeByLex

func (db *DBZSet) ZRangeByLex(ctx context.Context, key []byte, min []byte, max []byte, rangeType driver.RangeType, offset int, count int) ([][]byte, error)

ZRangeByLex scans the zset lexicographically

func (*DBZSet) ZRangeByScoreGeneric

func (db *DBZSet) ZRangeByScoreGeneric(ctx context.Context, key []byte, min int64, max int64,
	offset int, count int, reverse bool) ([]driver.ScorePair, error)

ZRangeByScoreGeneric is a generic function to scan zset with score. min and max must be inclusive if no limit, set offset = 0 and count<0

func (*DBZSet) ZRangeGeneric

func (db *DBZSet) ZRangeGeneric(ctx context.Context, key []byte, start int, stop int, reverse bool) ([]driver.ScorePair, error)

ZRangeGeneric is a generic function for scan zset. zrange/zrevrange index pos start,stop

func (*DBZSet) ZRank

func (db *DBZSet) ZRank(ctx context.Context, key []byte, member []byte) (int64, error)

func (*DBZSet) ZRem

func (db *DBZSet) ZRem(ctx context.Context, key []byte, members ...[]byte) (int64, error)

func (*DBZSet) ZRemRangeByLex

func (db *DBZSet) ZRemRangeByLex(ctx context.Context, key []byte, min []byte, max []byte, rangeType driver.RangeType) (int64, error)

ZRemRangeByLex remvoes members in [min, max] lexicographically

func (*DBZSet) ZRemRangeByRank

func (db *DBZSet) ZRemRangeByRank(ctx context.Context, key []byte, start int, stop int) (int64, error)

ZRemRangeByRank removes the member at range from start to stop.

func (*DBZSet) ZRemRangeByScore

func (db *DBZSet) ZRemRangeByScore(ctx context.Context, key []byte, min int64, max int64) (int64, error)

ZRemRangeByScore removes the data with score at [min, max]

func (*DBZSet) ZRevRange

func (db *DBZSet) ZRevRange(ctx context.Context, key []byte, start int, stop int) ([]driver.ScorePair, error)

func (*DBZSet) ZRevRangeByScore

func (db *DBZSet) ZRevRangeByScore(ctx context.Context, key []byte, min int64, max int64, offset int, count int) ([]driver.ScorePair, error)

ZRevRangeByScore gets the data with score at [min, max]

func (*DBZSet) ZRevRank

func (db *DBZSet) ZRevRank(ctx context.Context, key []byte, member []byte) (int64, error)

func (*DBZSet) ZScore

func (db *DBZSet) ZScore(ctx context.Context, key []byte, member []byte) (int64, error)

func (*DBZSet) ZUnionStore

func (db *DBZSet) ZUnionStore(ctx context.Context, destKey []byte, srcKeys [][]byte, weights []int64, aggregate []byte) (int64, error)

ZUnionStore unions the zsets and stores to dest zset.

type DataType

type DataType byte

DataType is defined for the different types

const (
	STRING DataType = iota
	LIST
	HASH
	SET
	ZSET
	BITMAP
)

for out use

func (DataType) String

func (d DataType) String() string

type GCChecker

type GCChecker struct {
	// contains filtered or unexported fields
}

func NewGCChecker

func NewGCChecker(opts *config.GCJobOptions, client *tikv.Client, leader *LeaderChecker, store *Storager) *GCChecker

func (*GCChecker) Run

func (m *GCChecker) Run(ctx context.Context)

type LeaderChecker

type LeaderChecker struct {
	// contains filtered or unexported fields
}

func NewLeaderChecker

func NewLeaderChecker(opts *config.LeaderJobOptions, client *tikv.Client, store *Storager) *LeaderChecker

func (*LeaderChecker) IsLeader

func (m *LeaderChecker) IsLeader(ctx context.Context) bool

IsLeader if check leader val format and lease not time out, is true

func (*LeaderChecker) Run

func (m *LeaderChecker) Run(ctx context.Context)

type Storager

type Storager struct {
	// contains filtered or unexported fields
}

Storager core store struct for server use like redis

func New

func New(opts *config.StoragerOptions) (store *Storager)

func (*Storager) Close

func (m *Storager) Close() (err error)

Close close tikv client

func (*Storager) FlushAll

func (m *Storager) FlushAll(ctx context.Context) (err error)

FlushAll will clear all data if use shared dist tikv , need prefix key to logic isolation use namespace/tenantId(appId/bizId);

func (*Storager) InitOpts

func (m *Storager) InitOpts(opts *config.StoragerOptions)

func (*Storager) Name

func (m *Storager) Name() string

func (*Storager) Open

func (store *Storager) Open(ctx context.Context) (err error)

func (*Storager) PrefixKey

func (m *Storager) PrefixKey() []byte

PrefixKey get the prefix key

func (*Storager) Select

func (m *Storager) Select(ctx context.Context, index int) (idb driver.IDB, err error)

Select chooses a database.

func (*Storager) SetPrefix

func (m *Storager) SetPrefix(prefix string)

SetPrefix set the prefix key.

type TTLChecker

type TTLChecker struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewTTLChecker

func NewTTLChecker(db *DB) *TTLChecker

func (*TTLChecker) Run

func (c *TTLChecker) Run(ctx context.Context)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL