Documentation
¶
Index ¶
- Constants
- func CalcPIndexesDelta(mgrUUID string, currPIndexes map[string]*PIndex, ...) (addPlanPIndexes []*PlanPIndex, removePIndexes []*PIndex)
- func CfgNodeDefsKey(kind string) string
- func CfgSetIndexDefs(cfg Cfg, indexDefs *IndexDefs, cas uint64) (uint64, error)
- func CfgSetNodeDefs(cfg Cfg, kind string, nodeDefs *NodeDefs, cas uint64) (uint64, error)
- func CfgSetPlanPIndexes(cfg Cfg, planPIndexes *PlanPIndexes, cas uint64) (uint64, error)
- func CheckVersion(cfg Cfg, myVersion string) (bool, error)
- func CouchbasePartitions(sourceType, sourceName, sourceUUID, sourceParams, server string) ([]string, error)
- func CountAlias(mgr *Manager, indexName, indexUUID string) (uint64, error)
- func CountBlevePIndexImpl(mgr *Manager, indexName, indexUUID string) (uint64, error)
- func DataSourcePartitions(sourceType, sourceName, sourceUUID, sourceParams, server string) ([]string, error)
- func DestFeedPartitions(sourceType, sourceName, sourceUUID, sourceParams, server string) ([]string, error)
- func ExponentialBackoffLoop(name string, f func() int, startSleepMS int, backoffFactor float32, ...)
- func FeedName(pindex *PIndex) string
- func NewBlackHolePIndexImpl(indexType, indexParams, path string, restart func()) (PIndexImpl, Dest, error)
- func NewBlevePIndexImpl(indexType, indexParams, path string, restart func()) (PIndexImpl, Dest, error)
- func NewManagerRESTRouter(mgr *Manager, staticDir, staticETag string, mr *MsgRing) (*mux.Router, error)
- func NewPIndexImpl(indexType, indexParams, path string, restart func()) (PIndexImpl, Dest, error)
- func NewUUID() string
- func OpenBlackHolePIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error)
- func OpenBlevePIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error)
- func OpenPIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error)
- func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool
- func PIndexPath(dataDir, pindexName string) string
- func ParsePIndexPath(dataDir, pindexPath string) (string, bool)
- func ParsePartitionsToVBucketIds(dests map[string]Dest) ([]uint16, error)
- func PlanPIndexName(indexDef *IndexDef, sourcePartitions string) string
- func PlanPIndexNodeCanRead(p *PlanPIndexNode) bool
- func PlanPIndexNodeCanWrite(p *PlanPIndexNode) bool
- func PlannerCheckVersion(cfg Cfg, version string) error
- func QueryAlias(mgr *Manager, indexName, indexUUID string, req []byte, res io.Writer) error
- func QueryBlevePIndexImpl(mgr *Manager, indexName, indexUUID string, req []byte, res io.Writer) error
- func RegisterFeedType(sourceType string, f *FeedType)
- func RegisterPIndexImplType(indexType string, t *PIndexImplType)
- func RewriteURL(to string, h http.Handler) http.Handler
- func SamePlanPIndex(a, b *PlanPIndex) bool
- func SamePlanPIndexes(a, b *PlanPIndexes) bool
- func StartDCPFeed(mgr *Manager, ...) error
- func StartTAPFeed(mgr *Manager, ...) error
- func StringsIntersectStrings(a, b []string) []string
- func StringsRemoveStrings(stringArr, removeArr []string) []string
- func StringsToMap(strsArr []string) map[string]bool
- func SubsetPlanPIndexes(a, b *PlanPIndexes) bool
- func SyncWorkReq(ch chan *WorkReq, op, msg string, obj interface{}) error
- func ValidateAlias(indexType, indexName, indexParams string) error
- func ValidateBlevePIndexImpl(indexType, indexName, indexParams string) error
- func VersionGTE(x, y string) bool
- type AliasParams
- type AliasParamsTarget
- type BlackHole
- func (t *BlackHole) Close() error
- func (t *BlackHole) ConsistencyWait(partition string, consistencyLevel string, consistencySeq uint64, ...) error
- func (t *BlackHole) Count(pindex *PIndex, cancelCh chan struct{}) (uint64, error)
- func (t *BlackHole) GetOpaque(partition string) (value []byte, lastSeq uint64, err error)
- func (t *BlackHole) OnDataDelete(partition string, key []byte, seq uint64) error
- func (t *BlackHole) OnDataUpdate(partition string, key []byte, seq uint64, val []byte) error
- func (t *BlackHole) OnSnapshotStart(partition string, snapStart, snapEnd uint64) error
- func (t *BlackHole) Query(pindex *PIndex, req []byte, w io.Writer, cancelCh chan struct{}) error
- func (t *BlackHole) Rollback(partition string, rollbackSeq uint64) error
- func (t *BlackHole) SetOpaque(partition string, value []byte) error
- type BleveClient
- func (r *BleveClient) Batch(b *bleve.Batch) error
- func (r *BleveClient) Close() error
- func (r *BleveClient) Delete(id string) error
- func (r *BleveClient) DeleteInternal(key []byte) error
- func (r *BleveClient) DocCount() (uint64, error)
- func (r *BleveClient) Document(id string) (*document.Document, error)
- func (r *BleveClient) DumpAll() chan interface{}
- func (r *BleveClient) DumpDoc(id string) chan interface{}
- func (r *BleveClient) DumpFields() chan interface{}
- func (r *BleveClient) Fields() ([]string, error)
- func (r *BleveClient) GetInternal(key []byte) ([]byte, error)
- func (r *BleveClient) Index(id string, data interface{}) error
- func (r *BleveClient) Mapping() *bleve.IndexMapping
- func (r *BleveClient) Search(req *bleve.SearchRequest) (*bleve.SearchResult, error)
- func (r *BleveClient) SetInternal(key, val []byte) error
- func (r *BleveClient) Stats() *bleve.IndexStat
- type BleveDest
- func (t *BleveDest) Close() error
- func (t *BleveDest) ConsistencyWait(partition string, consistencyLevel string, consistencySeq uint64, ...) error
- func (t *BleveDest) Count(pindex *PIndex, cancelCh chan struct{}) (uint64, error)
- func (t *BleveDest) GetOpaque(partition string) (value []byte, lastSeq uint64, err error)
- func (t *BleveDest) OnDataDelete(partition string, key []byte, seq uint64) error
- func (t *BleveDest) OnDataUpdate(partition string, key []byte, seq uint64, val []byte) error
- func (t *BleveDest) OnSnapshotStart(partition string, snapStart, snapEnd uint64) error
- func (t *BleveDest) Query(pindex *PIndex, req []byte, res io.Writer, cancelCh chan struct{}) error
- func (t *BleveDest) Rollback(partition string, rollbackSeq uint64) error
- func (t *BleveDest) SetOpaque(partition string, value []byte) error
- type BleveDestPartition
- func (t *BleveDestPartition) GetOpaque(bindex bleve.Index) ([]byte, uint64, error)
- func (t *BleveDestPartition) OnDataDelete(bindex bleve.Index, key []byte, seq uint64) error
- func (t *BleveDestPartition) OnDataUpdate(bindex bleve.Index, key []byte, seq uint64, val []byte) error
- func (t *BleveDestPartition) OnSnapshotStart(bindex bleve.Index, snapStart, snapEnd uint64) error
- func (t *BleveDestPartition) SetOpaque(bindex bleve.Index, value []byte) error
- type BleveQueryParams
- type Cfg
- type CfgCASError
- type CfgCB
- func (r *CfgCB) DataDelete(vbucketId uint16, key []byte, seq uint64, req *gomemcached.MCRequest) error
- func (r *CfgCB) DataUpdate(vbucketId uint16, key []byte, seq uint64, req *gomemcached.MCRequest) error
- func (c *CfgCB) Del(key string, cas uint64) error
- func (c *CfgCB) Get(key string, cas uint64) ([]byte, uint64, error)
- func (a *CfgCB) GetCredentials() (string, string)
- func (r *CfgCB) GetMetaData(vbucketId uint16) (value []byte, lastSeq uint64, err error)
- func (c *CfgCB) Load() error
- func (r *CfgCB) OnError(err error)
- func (c *CfgCB) Refresh() error
- func (r *CfgCB) Rollback(vbucketId uint16, rollbackSeq uint64) error
- func (c *CfgCB) Set(key string, val []byte, cas uint64) (uint64, error)
- func (r *CfgCB) SetMetaData(vbucketId uint16, value []byte) error
- func (r *CfgCB) SnapshotStart(vbucketId uint16, snapStart, snapEnd uint64, snapType uint32) error
- func (c *CfgCB) Subscribe(key string, ch chan CfgEvent) error
- type CfgEvent
- type CfgGetHandler
- type CfgMem
- type CfgMemEntry
- type CfgRefreshHandler
- type CfgSimple
- func (c *CfgSimple) Del(key string, cas uint64) error
- func (c *CfgSimple) Get(key string, cas uint64) ([]byte, uint64, error)
- func (c *CfgSimple) Load() error
- func (c *CfgSimple) Refresh() error
- func (c *CfgSimple) Set(key string, val []byte, cas uint64) (uint64, error)
- func (c *CfgSimple) Subscribe(key string, ch chan CfgEvent) error
- type ConsistencyParams
- type ConsistencyVector
- type CountHandler
- type CountPIndexHandler
- type CreateIndexHandler
- type DCPFeed
- func (t *DCPFeed) Close() error
- func (r *DCPFeed) DataDelete(vbucketId uint16, key []byte, seq uint64, req *gomemcached.MCRequest) error
- func (r *DCPFeed) DataUpdate(vbucketId uint16, key []byte, seq uint64, req *gomemcached.MCRequest) error
- func (t *DCPFeed) Dests() map[string]Dest
- func (r *DCPFeed) GetMetaData(vbucketId uint16) (value []byte, lastSeq uint64, err error)
- func (t *DCPFeed) Name() string
- func (r *DCPFeed) OnError(err error)
- func (r *DCPFeed) Rollback(vbucketId uint16, rollbackSeq uint64) error
- func (r *DCPFeed) SetMetaData(vbucketId uint16, value []byte) error
- func (r *DCPFeed) SnapshotStart(vbucketId uint16, snapStart, snapEnd uint64, snapType uint32) error
- func (t *DCPFeed) Start() error
- func (t *DCPFeed) Stats(w io.Writer) error
- type DCPFeedParams
- type DeleteIndexHandler
- type Dest
- func BasicPartitionFunc(partition string, key []byte, dests map[string]Dest) (Dest, error)
- func NewBleveDest(path string, bindex bleve.Index, restart func()) Dest
- func VBucketIdToPartitionDest(pf DestPartitionFunc, dests map[string]Dest, vbucketId uint16, key []byte) (partition string, dest Dest, err error)
- type DestFeed
- func (t *DestFeed) Close() error
- func (t *DestFeed) ConsistencyWait(partition string, consistencyLevel string, consistencySeq uint64, ...) error
- func (t *DestFeed) Count(pindex *PIndex, cancelCh chan struct{}) (uint64, error)
- func (t *DestFeed) Dests() map[string]Dest
- func (t *DestFeed) GetOpaque(partition string) (value []byte, lastSeq uint64, err error)
- func (t *DestFeed) Name() string
- func (t *DestFeed) OnDataDelete(partition string, key []byte, seq uint64) error
- func (t *DestFeed) OnDataUpdate(partition string, key []byte, seq uint64, val []byte) error
- func (t *DestFeed) OnSnapshotStart(partition string, snapStart, snapEnd uint64) error
- func (t *DestFeed) Query(pindex *PIndex, req []byte, w io.Writer, cancelCh chan struct{}) error
- func (t *DestFeed) Rollback(partition string, rollbackSeq uint64) error
- func (t *DestFeed) SetOpaque(partition string, value []byte) error
- func (t *DestFeed) Start() error
- func (t *DestFeed) Stats(w io.Writer) error
- type DestPartitionFunc
- type DestSourceParams
- type Feed
- type FeedPartitionsFunc
- type FeedStartFunc
- type FeedStatsHandler
- type FeedType
- type GetIndexHandler
- type GetLogHandler
- type IndexDef
- type IndexDefs
- type ListIndexHandler
- type Manager
- func (mgr *Manager) Cfg() Cfg
- func (mgr *Manager) ClosePIndex(pindex *PIndex) error
- func (mgr *Manager) CoveringPIndexes(indexName, indexUUID string, wantNode func(*PlanPIndexNode) bool) (localPIndexes []*PIndex, remotePlanPIndexes []*RemotePlanPIndex, err error)
- func (mgr *Manager) CreateIndex(...) error
- func (mgr *Manager) CurrentMaps() (map[string]Feed, map[string]*PIndex)
- func (mgr *Manager) DataDir() string
- func (mgr *Manager) DeleteIndex(indexName string) error
- func (mgr *Manager) GetIndexDefs(refresh bool) (*IndexDefs, map[string]*IndexDef, error)
- func (mgr *Manager) GetPIndex(pindexName string) *PIndex
- func (mgr *Manager) GetPlanPIndexes(refresh bool) (*PlanPIndexes, map[string][]*PlanPIndex, error)
- func (mgr *Manager) JanitorKick(msg string)
- func (mgr *Manager) JanitorLoop()
- func (mgr *Manager) JanitorNOOP(msg string)
- func (mgr *Manager) JanitorOnce(reason string) error
- func (mgr *Manager) Kick(msg string)
- func (mgr *Manager) LoadDataDir() error
- func (mgr *Manager) PIndexPath(pindexName string) string
- func (mgr *Manager) ParsePIndexPath(pindexPath string) (string, bool)
- func (mgr *Manager) PlannerKick(msg string)
- func (mgr *Manager) PlannerLoop()
- func (mgr *Manager) PlannerNOOP(msg string)
- func (mgr *Manager) PlannerOnce(reason string) (bool, error)
- func (mgr *Manager) RemovePIndex(pindex *PIndex) error
- func (mgr *Manager) SaveNodeDef(kind string, force bool) error
- func (mgr *Manager) Start(register string) error
- func (mgr *Manager) UUID() string
- type ManagerEventHandlers
- type ManagerKickHandler
- type ManagerMetaHandler
- type MetaDesc
- type MsgRing
- type NILFeed
- type NodeDef
- type NodeDefs
- type PIndex
- type PIndexImpl
- type PIndexImplType
- type PlanPIndex
- type PlanPIndexNode
- type PlanPIndexNodeRef
- type PlanPIndexNodeRefs
- type PlanPIndexes
- func CalcPlan(indexDefs *IndexDefs, nodeDefs *NodeDefs, planPIndexesPrev *PlanPIndexes, ...) (*PlanPIndexes, error)
- func CfgGetPlanPIndexes(cfg Cfg) (*PlanPIndexes, uint64, error)
- func NewPlanPIndexes(version string) *PlanPIndexes
- func PlannerGetPlanPIndexes(cfg Cfg, version string) (*PlanPIndexes, uint64, error)
- type PlanParams
- type QueryHandler
- type QueryPIndexHandler
- type RemotePlanPIndex
- type TAPFeed
- type TAPFeedParams
- type WorkReq
Constants ¶
const BLEVE_DEST_APPLY_BUF_SIZE_BYTES = 200000
const BLEVE_DEST_INITIAL_BUF_SIZE_BYTES = 20000
const FEED_BACKOFF_FACTOR = 1.5
const FEED_SLEEP_INIT_MS = 100
const FEED_SLEEP_MAX_MS = 10000
Default values for feed parameters.
const INDEX_DEFS_KEY = "indexDefs"
const JANITOR_CLOSE_PINDEX = "janitor_close_pindex"
const JANITOR_REMOVE_PINDEX = "janitor_remove_pindex"
const NODE_DEFS_KEY = "nodeDefs"
const NODE_DEFS_KNOWN = "known"
const NODE_DEFS_WANTED = "wanted"
const PINDEX_META_FILENAME string = "PINDEX_META"
const PLAN_PINDEXES_KEY = "planPIndexes"
const VERSION = "2.1.0"
NOTE: You *must* update VERSION if you change what's stored in the Cfg (such as the JSON/struct definitions or planning algorithms).
const VERSION_KEY = "version"
const WORK_KICK = "kick"
const WORK_NOOP = ""
Variables ¶
This section is empty.
Functions ¶
func CalcPIndexesDelta ¶
func CalcPIndexesDelta(mgrUUID string, currPIndexes map[string]*PIndex, wantedPlanPIndexes *PlanPIndexes) ( addPlanPIndexes []*PlanPIndex, removePIndexes []*PIndex)
Functionally determine the delta of which pindexes need creation and which should be shut down on our local node (mgrUUID).
func CfgNodeDefsKey ¶
func CfgSetIndexDefs ¶
func CfgSetNodeDefs ¶
func CfgSetPlanPIndexes ¶
func CfgSetPlanPIndexes(cfg Cfg, planPIndexes *PlanPIndexes, cas uint64) (uint64, error)
func CheckVersion ¶
Returns true if a given version is modern enough to modify the Cfg. Older versions (which are running with older JSON/struct defintions or planning algorithms) will see false from their CheckVersion()'s.
func CouchbasePartitions ¶
func CountBlevePIndexImpl ¶
func DataSourcePartitions ¶
func DestFeedPartitions ¶
func ExponentialBackoffLoop ¶
func ExponentialBackoffLoop(name string, f func() int, startSleepMS int, backoffFactor float32, maxSleepMS int)
Calls f() in a loop, sleeping in an exponential backoff if needed. The provided f() function should return < 0 to stop the loop; >= 0 to continue the loop, where > 0 means there was progress which allows an immediate retry of f() with no sleeping. A return of < 0 is useful when f() will never make any future progress.
func NewBlackHolePIndexImpl ¶
func NewBlackHolePIndexImpl(indexType, indexParams, path string, restart func()) ( PIndexImpl, Dest, error)
func NewBlevePIndexImpl ¶
func NewBlevePIndexImpl(indexType, indexParams, path string, restart func()) ( PIndexImpl, Dest, error)
func NewManagerRESTRouter ¶
func NewPIndexImpl ¶
func NewPIndexImpl(indexType, indexParams, path string, restart func()) ( PIndexImpl, Dest, error)
func OpenBlackHolePIndexImpl ¶
func OpenBlackHolePIndexImpl(indexType, path string, restart func()) ( PIndexImpl, Dest, error)
func OpenBlevePIndexImpl ¶
func OpenBlevePIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error)
func OpenPIndexImpl ¶
func OpenPIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error)
func PIndexMatchesPlan ¶
func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool
Returns true if both the PIndex meets the PlanPIndex, ignoring UUID.
func PIndexPath ¶
func ParsePIndexPath ¶
func PlanPIndexName ¶
NOTE: PlanPIndex.Name must be unique across the cluster and ideally functionally based off of the indexDef so that the SamePlanPIndex() comparison works even if concurrent planners are racing to calculate plans.
NOTE: We can't use sourcePartitions directly as part of a PlanPIndex.Name suffix because in vbucket/hash partitioning the string would be too long -- since PIndexes might use PlanPIndex.Name for filesystem paths.
func PlanPIndexNodeCanRead ¶
func PlanPIndexNodeCanRead(p *PlanPIndexNode) bool
func PlanPIndexNodeCanWrite ¶
func PlanPIndexNodeCanWrite(p *PlanPIndexNode) bool
func PlannerCheckVersion ¶
func QueryAlias ¶
func QueryBlevePIndexImpl ¶
func RegisterFeedType ¶
func RegisterPIndexImplType ¶
func RegisterPIndexImplType(indexType string, t *PIndexImplType)
func SamePlanPIndex ¶
func SamePlanPIndex(a, b *PlanPIndex) bool
Returns true if both PlanPIndex are the same, ignoring PlanPIndex.UUID.
func SamePlanPIndexes ¶
func SamePlanPIndexes(a, b *PlanPIndexes) bool
Returns true if both PlanPIndexes are the same, where we ignore any differences in UUID or ImplVersion.
func StartDCPFeed ¶
func StartTAPFeed ¶
func StringsIntersectStrings ¶
StringsIntersectStrings returns a brand new array that has the intersection of a and b.
func StringsRemoveStrings ¶
StringsRemoveStrings returns a copy of stringArr, but with some strings removed, keeping the same order as stringArr.
func StringsToMap ¶
func SubsetPlanPIndexes ¶
func SubsetPlanPIndexes(a, b *PlanPIndexes) bool
Returns true if PlanPIndex children in a are a subset of those in b, using SamePlanPIndex() for sameness comparion.
func SyncWorkReq ¶
func ValidateAlias ¶
func ValidateBlevePIndexImpl ¶
func VersionGTE ¶
Compares two dotted versioning strings, like "1.0.1" and "1.2.3". Returns true when x >= y.
Types ¶
type AliasParams ¶
type AliasParams struct {
Targets map[string]*AliasParamsTarget `json:"targets"` // Keyed by indexName.
}
AliasParams holds the definition for a user-defined index alias. A user-defined index alias can be used as a level of indirection (the "LastQuartersSales" alias points currently to the "2014-Q3-Sales" index, but the administrator might repoint it in the future without changing the application) or to scatter-gather or fan-out a query across multiple real indexes (e.g., to query across customer records, product catalog, call-center records, etc, in one shot).
type AliasParamsTarget ¶
type AliasParamsTarget struct {
IndexUUID string `json:"indexUUID"` // Optional.
}
type BlackHole ¶
type BlackHole struct {
// contains filtered or unexported fields
}
Implements both Dest and PIndexImpl interfaces.
func (*BlackHole) ConsistencyWait ¶
func (*BlackHole) OnDataDelete ¶
func (*BlackHole) OnDataUpdate ¶
func (*BlackHole) OnSnapshotStart ¶
type BleveClient ¶
type BleveClient struct {
QueryURL string
CountURL string
Consistency *ConsistencyParams
}
BleveClient implements the Search() and DocCount() subset of the bleve.Index interface by accessing a remote cbft server via REST protocol. This allows callers to add a BleveClient as a target of a bleve.IndexAlias, and implements cbft protocol features like query consistency and auth.
TODO: Implement propagating auth info in BleveClient.
func (*BleveClient) Close ¶
func (r *BleveClient) Close() error
func (*BleveClient) Delete ¶
func (r *BleveClient) Delete(id string) error
func (*BleveClient) DeleteInternal ¶
func (r *BleveClient) DeleteInternal(key []byte) error
func (*BleveClient) DocCount ¶
func (r *BleveClient) DocCount() (uint64, error)
func (*BleveClient) Document ¶
func (r *BleveClient) Document(id string) (*document.Document, error)
func (*BleveClient) DumpAll ¶
func (r *BleveClient) DumpAll() chan interface{}
func (*BleveClient) DumpDoc ¶
func (r *BleveClient) DumpDoc(id string) chan interface{}
func (*BleveClient) DumpFields ¶
func (r *BleveClient) DumpFields() chan interface{}
func (*BleveClient) Fields ¶
func (r *BleveClient) Fields() ([]string, error)
func (*BleveClient) GetInternal ¶
func (r *BleveClient) GetInternal(key []byte) ([]byte, error)
func (*BleveClient) Index ¶
func (r *BleveClient) Index(id string, data interface{}) error
func (*BleveClient) Mapping ¶
func (r *BleveClient) Mapping() *bleve.IndexMapping
func (*BleveClient) Search ¶
func (r *BleveClient) Search(req *bleve.SearchRequest) (*bleve.SearchResult, error)
func (*BleveClient) SetInternal ¶
func (r *BleveClient) SetInternal(key, val []byte) error
func (*BleveClient) Stats ¶
func (r *BleveClient) Stats() *bleve.IndexStat
type BleveDest ¶
type BleveDest struct {
// contains filtered or unexported fields
}
func (*BleveDest) ConsistencyWait ¶
func (*BleveDest) OnDataDelete ¶
func (*BleveDest) OnDataUpdate ¶
func (*BleveDest) OnSnapshotStart ¶
type BleveDestPartition ¶
type BleveDestPartition struct {
// contains filtered or unexported fields
}
Used to track state for a single partition.
func (*BleveDestPartition) OnDataDelete ¶
func (*BleveDestPartition) OnDataUpdate ¶
func (*BleveDestPartition) OnSnapshotStart ¶
func (t *BleveDestPartition) OnSnapshotStart(bindex bleve.Index, snapStart, snapEnd uint64) error
type BleveQueryParams ¶
type BleveQueryParams struct {
Query *bleve.SearchRequest `json:"query"`
Consistency *ConsistencyParams `json:"consistency"`
Timeout int64 `json:"timeout"`
}
type Cfg ¶
type Cfg interface {
// Get retrieves an entry from the Cfg. A zero cas means don't do
// a CAS match on Get(), and a non-zero cas value means the Get()
// will succeed only if the CAS matches.
Get(key string, cas uint64) (val []byte, casSuccess uint64, err error)
// Set creates or updates an entry in the Cfg. A non-zero cas
// that does not match will result in an error. A zero cas means
// the Set() operation must be an entry creation, where a zero cas
// Set() will error if the entry already exists.
Set(key string, val []byte, cas uint64) (casSuccess uint64, err error)
// Del removes an entry from the Cfg. A non-zero cas that does
// not match will result in an error. A zero cas means a CAS
// match will be skipped, so that clients can perform a
// "don't-care, out-of-the-blue" deletion.
Del(key string, cas uint64) error
// Subscribe allows clients to receive events on changes to a key.
// During a deletion event, the CfgEvent.CAS field will be 0.
Subscribe(key string, ch chan CfgEvent) error
// Refresh forces the Cfg implementation to reload from its
// backend-specific data source, clearing any locally cached data.
// Any subscribers will receive events on a Refresh, where it's up
// to subscribers to detect if there were actual changes or not.
Refresh() error
}
Cfg is the interface that configuration providers must implement.
type CfgCASError ¶
type CfgCASError struct{}
The error used on mismatches of CAS (compare and set/swap) values.
func (*CfgCASError) Error ¶
func (e *CfgCASError) Error() string
type CfgCB ¶
type CfgCB struct {
// contains filtered or unexported fields
}
CfgCB is an implementation of Cfg that uses a couchbase bucket.
TODO: This current implementation is race-y! Instead of storing everything as a single uber key/value, we should instead be storing individual key/value's on every get/set/del operation.
func (*CfgCB) DataDelete ¶
func (*CfgCB) DataUpdate ¶
func (*CfgCB) GetCredentials ¶
func (*CfgCB) GetMetaData ¶
func (*CfgCB) SnapshotStart ¶
type CfgGetHandler ¶
type CfgGetHandler struct {
// contains filtered or unexported fields
}
func NewCfgGetHandler ¶
func NewCfgGetHandler(mgr *Manager) *CfgGetHandler
func (*CfgGetHandler) ServeHTTP ¶
func (h *CfgGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type CfgMem ¶
type CfgMem struct {
CASNext uint64
Entries map[string]*CfgMemEntry
// contains filtered or unexported fields
}
type CfgMemEntry ¶
type CfgRefreshHandler ¶
type CfgRefreshHandler struct {
// contains filtered or unexported fields
}
func NewCfgRefreshHandler ¶
func NewCfgRefreshHandler(mgr *Manager) *CfgRefreshHandler
func (*CfgRefreshHandler) ServeHTTP ¶
func (h *CfgRefreshHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type CfgSimple ¶
type CfgSimple struct {
// contains filtered or unexported fields
}
func NewCfgSimple ¶
type ConsistencyParams ¶
type ConsistencyParams struct {
// A Level value of "" means stale is ok; "at_plus" means we need
// consistency at least at or beyond the consistency vector but
// not before.
Level string `json:"level"`
// Keyed by indexName.
Vectors map[string]ConsistencyVector `json:"vectors"`
}
type CountHandler ¶
type CountHandler struct {
// contains filtered or unexported fields
}
func NewCountHandler ¶
func NewCountHandler(mgr *Manager) *CountHandler
func (*CountHandler) ServeHTTP ¶
func (h *CountHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type CountPIndexHandler ¶
type CountPIndexHandler struct {
// contains filtered or unexported fields
}
func NewCountPIndexHandler ¶
func NewCountPIndexHandler(mgr *Manager) *CountPIndexHandler
func (*CountPIndexHandler) ServeHTTP ¶
func (h *CountPIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type CreateIndexHandler ¶
type CreateIndexHandler struct {
// contains filtered or unexported fields
}
func NewCreateIndexHandler ¶
func NewCreateIndexHandler(mgr *Manager) *CreateIndexHandler
func (*CreateIndexHandler) ServeHTTP ¶
func (h *CreateIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type DCPFeed ¶
type DCPFeed struct {
// contains filtered or unexported fields
}
A DCPFeed implements both Feed and cbdatasource.Receiver interfaces.
func NewDCPFeed ¶
func (*DCPFeed) DataDelete ¶
func (*DCPFeed) DataUpdate ¶
func (*DCPFeed) GetMetaData ¶
func (*DCPFeed) SnapshotStart ¶
type DCPFeedParams ¶
type DCPFeedParams struct {
AuthUser string `json:"authUser"` // May be "" for no auth.
AuthPassword string `json:"authPassword"`
// Factor (like 1.5) to increase sleep time between retries
// in connecting to a cluster manager node.
ClusterManagerBackoffFactor float32 `json:"clusterManagerBackoffFactor"`
// Initial sleep time (millisecs) before first retry to cluster manager.
ClusterManagerSleepInitMS int `json:"clusterManagerSleepInitMS"`
// Maximum sleep time (millisecs) between retries to cluster manager.
ClusterManagerSleepMaxMS int `json:"clusterManagerSleepMaxMS"`
// Factor (like 1.5) to increase sleep time between retries
// in connecting to a data manager node.
DataManagerBackoffFactor float32 `json:"dataManagerBackoffFactor"`
// Initial sleep time (millisecs) before first retry to data manager.
DataManagerSleepInitMS int `json:"dataManagerSleepInitMS"`
// Maximum sleep time (millisecs) between retries to data manager.
DataManagerSleepMaxMS int `json:"dataManagerSleepMaxMS"`
// Buffer size in bytes provided for UPR flow control.
FeedBufferSizeBytes uint32 `json:"feedBufferSizeBytes"`
// Used for UPR flow control and buffer-ack messages when this
// percentage of FeedBufferSizeBytes is reached.
FeedBufferAckThreshold float32 `json:"feedBufferAckThreshold"`
}
func (*DCPFeedParams) GetCredentials ¶
func (d *DCPFeedParams) GetCredentials() (string, string)
type DeleteIndexHandler ¶
type DeleteIndexHandler struct {
// contains filtered or unexported fields
}
func NewDeleteIndexHandler ¶
func NewDeleteIndexHandler(mgr *Manager) *DeleteIndexHandler
func (*DeleteIndexHandler) ServeHTTP ¶
func (h *DeleteIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type Dest ¶
type Dest interface {
// Invoked by PIndex.Close().
Close() error
// Invoked when there's a new mutation from a data source for a
// partition. Dest implementation is responsible for making its
// own copies of the key and val data.
OnDataUpdate(partition string, key []byte, seq uint64, val []byte) error
// Invoked by the data source when there's a data deletion in a
// partition. Dest implementation is responsible for making its
// own copies of the key data.
OnDataDelete(partition string, key []byte, seq uint64) error
// An callback invoked by the data source when there's a start of
// a new snapshot for a partition. The Receiver implementation,
// for example, might choose to optimize persistence perhaps by
// preparing a batch write to application-specific storage.
OnSnapshotStart(partition string, snapStart, snapEnd uint64) error
// The Dest implementation should persist the value parameter of
// SetOpaque() for retrieval during some future call to
// GetOpaque() by the system. The metadata value should be
// considered "in-stream", or as part of the sequence history of
// mutations. That is, a later Rollback() to some previous
// sequence number for a particular partition should rollback
// both persisted metadata and regular data. The Dest
// implementation should make its own copy of the value data.
SetOpaque(partition string, value []byte) error
// GetOpaque() should return the opaque value previously
// provided by an earlier call to SetOpaque(). If there was no
// previous call to SetOpaque(), such as in the case of a brand
// new instance of a Dest (as opposed to a restarted or reloaded
// Dest), the Dest should return (nil, 0, nil) for (value,
// lastSeq, err), respectively. The lastSeq should be the last
// sequence number received and persisted during calls to the
// Dest's OnDataUpdate() & OnDataDelete() methods.
GetOpaque(partition string) (value []byte, lastSeq uint64, err error)
// Invoked by when the datasource signals a rollback during dest
// initialization. Note that both regular data and opaque data
// should be rolled back to at a maximum of the rollbackSeq. Of
// note, the Dest is allowed to rollback even further, even all
// the way back to the start or to zero.
Rollback(partition string, rollbackSeq uint64) error
// Blocks until the Dest has reached the desired consistency for
// the partition or until the cancelCh is closed by some goroutine
// related to the calling goroutine.
ConsistencyWait(partition string,
consistencyLevel string,
consistencySeq uint64,
cancelCh chan struct{}) error
// Counts the underlying pindex implementation.
Count(pindex *PIndex, cancelCh chan struct{}) (uint64, error)
// Queries the underlying pindex implementation, blocking if
// needed for the Dest to reach the desired consistency.
Query(pindex *PIndex, req []byte, w io.Writer,
cancelCh chan struct{}) error
}
func BasicPartitionFunc ¶
This basic partition func first tries a direct lookup by partition string, else it tries the "" partition.
type DestFeed ¶
type DestFeed struct {
// contains filtered or unexported fields
}
A DestFeed implements both the Feed and Dest interfaces, for chainability; and is also useful for testing.
func NewDestFeed ¶
func NewDestFeed(name string, pf DestPartitionFunc, dests map[string]Dest) *DestFeed
func (*DestFeed) ConsistencyWait ¶
func (*DestFeed) OnDataDelete ¶
func (*DestFeed) OnDataUpdate ¶
func (*DestFeed) OnSnapshotStart ¶
type DestPartitionFunc ¶
type DestSourceParams ¶
type DestSourceParams struct {
NumPartitions int `json:"numPartitions"`
}
type Feed ¶
type Feed interface {
Name() string
Start() error
Close() error
Dests() map[string]Dest // Key is partition identifier.
// Writes stats as JSON to the given writer.
Stats(io.Writer) error
}
func CalcFeedsDelta ¶
func CalcFeedsDelta(nodeUUID string, planPIndexes *PlanPIndexes, currFeeds map[string]Feed, pindexes map[string]*PIndex) ( addFeeds [][]*PIndex, removeFeeds []Feed)
Functionally determine the delta of which feeds need creation and which should be shut down.
type FeedPartitionsFunc ¶
type FeedStartFunc ¶
type FeedStatsHandler ¶
type FeedStatsHandler struct {
// contains filtered or unexported fields
}
func NewFeedStatsHandler ¶
func NewFeedStatsHandler(mgr *Manager) *FeedStatsHandler
func (*FeedStatsHandler) ServeHTTP ¶
func (h *FeedStatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type FeedType ¶
type FeedType struct {
Start FeedStartFunc
Partitions FeedPartitionsFunc
Public bool
Description string
StartSample interface{}
}
type GetIndexHandler ¶
type GetIndexHandler struct {
// contains filtered or unexported fields
}
func NewGetIndexHandler ¶
func NewGetIndexHandler(mgr *Manager) *GetIndexHandler
func (*GetIndexHandler) ServeHTTP ¶
func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type GetLogHandler ¶
type GetLogHandler struct {
// contains filtered or unexported fields
}
func NewGetLogHandler ¶
func NewGetLogHandler(mr *MsgRing) *GetLogHandler
func (*GetLogHandler) ServeHTTP ¶
func (h *GetLogHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type IndexDef ¶
type IndexDef struct {
Type string `json:"type"` // Ex: "bleve", "alias", "blackhole", etc.
Name string `json:"name"`
UUID string `json:"uuid"`
Params string `json:"params"`
SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"`
SourceUUID string `json:"sourceUUID"`
SourceParams string `json:"sourceParams"` // Optional connection info.
PlanParams PlanParams `json:"planParams"`
}
type IndexDefs ¶
type IndexDefs struct {
// IndexDefs.UUID changes whenever any child IndexDef changes.
UUID string `json:"uuid"`
IndexDefs map[string]*IndexDef `json:"indexDefs"` // Key is IndexDef.Name.
ImplVersion string `json:"implVersion"` // See VERSION.
}
func NewIndexDefs ¶
type ListIndexHandler ¶
type ListIndexHandler struct {
// contains filtered or unexported fields
}
func NewListIndexHandler ¶
func NewListIndexHandler(mgr *Manager) *ListIndexHandler
func (*ListIndexHandler) ServeHTTP ¶
func (h *ListIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
func NewManager ¶
func (*Manager) ClosePIndex ¶
func (*Manager) CoveringPIndexes ¶
func (mgr *Manager) CoveringPIndexes(indexName, indexUUID string, wantNode func(*PlanPIndexNode) bool) ( localPIndexes []*PIndex, remotePlanPIndexes []*RemotePlanPIndex, err error)
Returns a non-overlapping, disjoint set (or cut) of PIndexes (either local or remote) that cover all the partitons of an index so that the caller can perform scatter/gather queries, etc. Only PlanPIndexes on wanted nodes that pass the wantNode filter will be returned.
TODO: Perhaps need a tighter check around indexUUID, as the current implementation might have a race where old pindexes with a matching (but outdated) indexUUID might be chosen.
TODO: This implementation currently always favors the local node's pindex, but should it? Perhaps a remote node is more up-to-date than the local pindex?
TODO: We should favor the most up-to-date node rather than the first one that we run into here? But, perhaps the most up-to-date node is also the most overloaded? Or, perhaps the planner may be trying to rebalance away the most up-to-date node and hitting it with load just makes the rebalance take longer?
func (*Manager) CreateIndex ¶
func (mgr *Manager) CreateIndex(sourceType, sourceName, sourceUUID, sourceParams, indexType, indexName, indexParams string, planParams PlanParams) error
Creates a logical index, which might be comprised of many PIndex objects.
func (*Manager) CurrentMaps ¶
Returns a snapshot copy of the current feeds and pindexes.
func (*Manager) DeleteIndex ¶
Deletes a logical index, which might be comprised of many PIndex objects.
func (*Manager) GetIndexDefs ¶
Returns read-only snapshot of the IndexDefs, also with IndexDef's organized by name. Use refresh of true to force a read from Cfg.
func (*Manager) GetPlanPIndexes ¶
func (mgr *Manager) GetPlanPIndexes(refresh bool) ( *PlanPIndexes, map[string][]*PlanPIndex, error)
Returns read-only snapshot of the PlanPIndexes, also with PlanPIndex's organized by IndexName. Use refresh of true to force a read from Cfg.
func (*Manager) JanitorKick ¶
JanitorKick synchronously kicks the manager's janitor, if any.
func (*Manager) JanitorLoop ¶
func (mgr *Manager) JanitorLoop()
JanitorLoop is the main loop for the janitor.
func (*Manager) JanitorNOOP ¶
JanitorNOOP sends a synchronous NOOP request to the manager's janitor, if any.
func (*Manager) JanitorOnce ¶
func (*Manager) LoadDataDir ¶
Walk the data dir and register pindexes.
func (*Manager) PIndexPath ¶
func (*Manager) ParsePIndexPath ¶
func (*Manager) PlannerKick ¶
PlannerKick synchronously kicks the manager's planner, if any.
func (*Manager) PlannerLoop ¶
func (mgr *Manager) PlannerLoop()
PlannerLoop is the main loop for the planner.
func (*Manager) PlannerNOOP ¶
PlannerNOOP sends a synchronous NOOP request to the manager's planner, if any.
func (*Manager) RemovePIndex ¶
type ManagerEventHandlers ¶
type ManagerKickHandler ¶
type ManagerKickHandler struct {
// contains filtered or unexported fields
}
func NewManagerKickHandler ¶
func NewManagerKickHandler(mgr *Manager) *ManagerKickHandler
func (*ManagerKickHandler) ServeHTTP ¶
func (h *ManagerKickHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type ManagerMetaHandler ¶
type ManagerMetaHandler struct {
// contains filtered or unexported fields
}
func NewManagerMetaHandler ¶
func NewManagerMetaHandler(mgr *Manager) *ManagerMetaHandler
func (*ManagerMetaHandler) ServeHTTP ¶
func (h *ManagerMetaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type MetaDesc ¶
type MetaDesc struct {
Description string `json:"description"`
StartSample interface{} `json:"startSample"`
}
type MsgRing ¶
type NILFeed ¶
type NILFeed struct {
// contains filtered or unexported fields
}
A NILFeed never feeds any data to its dests. It's useful for testing and for pindexes that are actually primary data sources.
type NodeDefs ¶
type NodeDefs struct {
// NodeDefs.UUID changes whenever any child NodeDef changes.
UUID string `json:"uuid"`
NodeDefs map[string]*NodeDef `json:"nodeDefs"` // Key is NodeDef.HostPort.
ImplVersion string `json:"implVersion"` // See VERSION.
}
func NewNodeDefs ¶
type PIndex ¶
type PIndex struct {
Name string `json:"name"`
UUID string `json:"uuid"`
IndexType string `json:"indexType"`
IndexName string `json:"indexName"`
IndexUUID string `json:"indexUUID"`
IndexParams string `json:"indexParams"`
SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"`
SourceUUID string `json:"sourceUUID"`
SourceParams string `json:"sourceParams"`
SourcePartitions string `json:"sourcePartitions"`
Path string `json:"-"` // Transient, not persisted.
Impl PIndexImpl `json:"-"` // Transient, not persisted.
Dest Dest `json:"-"` // Transient, not persisted.
// contains filtered or unexported fields
}
func OpenPIndex ¶
NOTE: Path argument must be a directory.
type PIndexImpl ¶
type PIndexImpl interface {
Close() error
}
type PIndexImplType ¶
type PIndexImplType struct {
Validate func(indexType, indexName, indexParams string) error
New func(indexType, indexParams, path string, restart func()) (
PIndexImpl, Dest, error)
Open func(indexType, path string, restart func()) (
PIndexImpl, Dest, error)
Count func(mgr *Manager, indexName, indexUUID string) (
uint64, error)
Query func(mgr *Manager, indexName, indexUUID string,
req []byte, res io.Writer) error
Description string
StartSample interface{}
}
func PIndexImplTypeForIndex ¶
func PIndexImplTypeForIndex(cfg Cfg, indexName string) (*PIndexImplType, error)
type PlanPIndex ¶
type PlanPIndex struct {
Name string `json:"name"` // Stable & unique cluster wide.
UUID string `json:"uuid"`
IndexType string `json:"indexType"` // See IndexDef.Type.
IndexName string `json:"indexName"` // See IndexDef.Name.
IndexUUID string `json:"indexUUID"` // See IndefDef.UUID.
IndexParams string `json:"indexParams"` // See IndexDef.Params.
SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"`
SourceUUID string `json:"sourceUUID"`
SourceParams string `json:"sourceParams"` // Optional connection info.
SourcePartitions string `json:"sourcePartitions"`
Nodes map[string]*PlanPIndexNode `json:"nodes"` // Keyed by NodeDef.UUID.
}
type PlanPIndexNode ¶
type PlanPIndexNodeRef ¶
type PlanPIndexNodeRef struct {
UUID string
Node *PlanPIndexNode
}
type PlanPIndexNodeRefs ¶
type PlanPIndexNodeRefs []*PlanPIndexNodeRef
func (PlanPIndexNodeRefs) Len ¶
func (pms PlanPIndexNodeRefs) Len() int
func (PlanPIndexNodeRefs) Less ¶
func (pms PlanPIndexNodeRefs) Less(i, j int) bool
func (PlanPIndexNodeRefs) Swap ¶
func (pms PlanPIndexNodeRefs) Swap(i, j int)
type PlanPIndexes ¶
type PlanPIndexes struct {
// PlanPIndexes.UUID changes whenever any child PlanPIndex changes.
UUID string `json:"uuid"`
PlanPIndexes map[string]*PlanPIndex `json:"planPIndexes"` // Key is PlanPIndex.Name.
ImplVersion string `json:"implVersion"` // See VERSION.
Warnings map[string][]string `json:"warnings"` // Key is IndexDef.Name.
}
func CalcPlan ¶
func CalcPlan(indexDefs *IndexDefs, nodeDefs *NodeDefs, planPIndexesPrev *PlanPIndexes, version, server string) ( *PlanPIndexes, error)
Split logical indexes into PIndexes and assign PIndexes to nodes.
func CfgGetPlanPIndexes ¶
func CfgGetPlanPIndexes(cfg Cfg) (*PlanPIndexes, uint64, error)
func NewPlanPIndexes ¶
func NewPlanPIndexes(version string) *PlanPIndexes
func PlannerGetPlanPIndexes ¶
func PlannerGetPlanPIndexes(cfg Cfg, version string) (*PlanPIndexes, uint64, error)
type PlanParams ¶
type PlanParams struct {
MaxPartitionsPerPIndex int `json:"maxPartitionsPerPIndex"`
// The first copy is not counted as a replica. For example, a
// NumReplicas setting of 2 means there should be a primary and 2
// replicas... so 3 copies in total. A NumReplicas of 0 means
// just the first, primary copy only.
NumReplicas int `json:"numReplicas"`
HierarchyRules blance.HierarchyRules `json:"hierarchyRules"`
}
type QueryHandler ¶
type QueryHandler struct {
// contains filtered or unexported fields
}
func NewQueryHandler ¶
func NewQueryHandler(mgr *Manager) *QueryHandler
func (*QueryHandler) ServeHTTP ¶
func (h *QueryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type QueryPIndexHandler ¶
type QueryPIndexHandler struct {
// contains filtered or unexported fields
}
func NewQueryPIndexHandler ¶
func NewQueryPIndexHandler(mgr *Manager) *QueryPIndexHandler
func (*QueryPIndexHandler) ServeHTTP ¶
func (h *QueryPIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type RemotePlanPIndex ¶
type RemotePlanPIndex struct {
PlanPIndex *PlanPIndex
NodeDef *NodeDef
}
type TAPFeed ¶
type TAPFeed struct {
// contains filtered or unexported fields
}
A TAPFeed uses TAP protocol to dump data from a couchbase data source.
func NewTAPFeed ¶
type TAPFeedParams ¶
Source Files
¶
- cfg.go
- cfg_cb.go
- cfg_mem.go
- cfg_simple.go
- defs.go
- dest.go
- dest_feed.go
- feed.go
- feed_dcp.go
- feed_nil.go
- feed_tap.go
- http.go
- manager.go
- manager_api.go
- manager_janitor.go
- manager_planner.go
- misc.go
- msg_ring.go
- pindex.go
- pindex_impl.go
- pindex_impl_alias.go
- pindex_impl_blackhole.go
- pindex_impl_bleve.go
- remote.go
- rest.go
- rest_create_index.go
- rest_delete_index.go
- rest_index.go
- rest_log.go
- rest_manage.go
- rest_meta.go
- version.go
- work.go

