computer

package
v3.26.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 12, 2022 License: Apache-2.0 Imports: 9 Imported by: 0

Documentation

Overview

Package computer contains the compute-specific portions of the DAX architecture. In general, this is a dumb FeatureBase node (or service) which essentially contains the Executor and its interaction with the underlying data.

Index

Constants

View Source 
const (
	EncodeTypeJSON string = "json"
)

Variables

This section is empty.

Functions

func MarshalLogMessage 

func MarshalLogMessage(msg LogMessage, encode string) ([]byte, error)

MarshalLogMessage serializes the log message and prepends additional encoding information to each message. Currently, we prepend three bytes to each log message: byte[0]: encodeVersion - this is currently a constant within the code. If we modify structs such that they encode differently, we'll have to change the constant and keep previous versions of structs for deserialization. byte[1]: encodeType (e.g. "json", etc.) byte[2]: logMessageType

If we get into a situation where we want more flexibility in these message header bytes—for example, if we want to use more than three bytes—we could do something with the first bit of the encodeVersion: if it's 1, that could indicate that there are additional header bytes, and the following seven bits could indicate how many.

func NewSnapshotReadWriter 

func NewSnapshotReadWriter(ss SnapshotService) *snapshotReadWriter

func NewWriteLogReadWriter 

func NewWriteLogReadWriter(wls WriteLogService) *writeLogReadWriter

Types

type FieldKeyMap 

type FieldKeyMap struct {
	TableKey   dax.TableKey      `json:"table-key"`
	Field      dax.FieldName     `json:"field"`
	StringToID map[string]uint64 `json:"string-to-id"`
}

type FieldKeyReader 

type FieldKeyReader interface {
	Open() error
	Read() (FieldKeyMap, error)
	Close() error
}

type ImportMessage 

type ImportMessage struct {
	LogMessage `json:"-"`

	Table      string   `json:"table"`
	Field      string   `json:"field"`
	Partition  int      `json:"partition"`
	Shard      uint64   `json:"shard"`
	RowIDs     []uint64 `json:"row-ids"`
	ColumnIDs  []uint64 `json:"column-ids"`
	RowKeys    []string `json:"row-keys"`
	ColumnKeys []string `json:"column-keys"`
	Timestamps []int64  `json:"timestamps"`
	Clear      bool     `json:"clear"`

	// options
	IgnoreKeyCheck bool `json:"ignore-key-check"`
	Presorted      bool `json:"presorted"`
}

type ImportRoaringMessage 

type ImportRoaringMessage struct {
	LogMessage `json:"-"`

	Table           string            `json:"table"`
	Field           string            `json:"field"`
	Partition       int               `json:"partition"`
	Shard           uint64            `json:"shard"`
	Clear           bool              `json:"clear"`
	Action          string            `json:"action"` // [set, clear, overwrite]
	Block           int               `json:"block"`
	Views           map[string][]byte `json:"views"`
	UpdateExistence bool              `json:"update-existence"`
}

type ImportRoaringShardMessage 

type ImportRoaringShardMessage struct {
	LogMessage `json:"-"`

	Table     string          `json:"table"`
	Partition int             `json:"partition"`
	Shard     uint64          `json:"shard"`
	Views     []RoaringUpdate `json:"views"`
}

type ImportValueMessage 

type ImportValueMessage struct {
	LogMessage `json:"-"`

	Table           string      `json:"table"`
	Field           string      `json:"field"`
	Partition       int         `json:"partition"`
	Shard           uint64      `json:"shard"`
	ColumnIDs       []uint64    `json:"column-ids"`
	ColumnKeys      []string    `json:"column-keys"`
	Values          []int64     `json:"values"`
	FloatValues     []float64   `json:"float-values"`
	TimestampValues []time.Time `json:"timestamp-values"`
	StringValues    []string    `json:"string-values"`
	Clear           bool        `json:"clear"`

	// options
	IgnoreKeyCheck bool `json:"ignore-key-check"`
	Presorted      bool `json:"presorted"`
}

type LogMessage 

type LogMessage interface{}

LogMessage is implemented by a variety of types which can be serialized as messages to the WriteLogger.

func UnmarshalLogMessage 

func UnmarshalLogMessage(b []byte) (LogMessage, error)

UnmarshalLogMessage deserializes the log message based on the log message type info.

type NopFieldKeyReader 

type NopFieldKeyReader struct{}

NopFieldKeyReader is a no-op implementation of the FieldKeyReader interface.

func NewNopFieldKeyReader 

func NewNopFieldKeyReader() *NopFieldKeyReader

func (*NopFieldKeyReader) Close 

func (r *NopFieldKeyReader) Close() error

func (*NopFieldKeyReader) Open 

func (r *NopFieldKeyReader) Open() error

func (*NopFieldKeyReader) Read 

func (r *NopFieldKeyReader) Read() (FieldKeyMap, error)

type NopShardReader 

type NopShardReader struct{}

NopShardReader is a no-op implementation of the ShardReader interface.

func NewNopShardReader 

func NewNopShardReader() *NopShardReader

func (*NopShardReader) Close 

func (r *NopShardReader) Close() error

func (*NopShardReader) Open 

func (r *NopShardReader) Open() error

func (*NopShardReader) Read 

func (r *NopShardReader) Read() (LogMessage, error)

type NopSnapshotReadWriter 

type NopSnapshotReadWriter struct{}

NopSnapshotReadWriter is a no-op implementation of the SnapshotReadWriter interface.

func NewNopSnapshotReadWriter 

func NewNopSnapshotReadWriter() *NopSnapshotReadWriter

func (*NopSnapshotReadWriter) ReadFieldKeys 

func (w *NopSnapshotReadWriter) ReadFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) (io.ReadCloser, error)

func (*NopSnapshotReadWriter) ReadShardData 

func (w *NopSnapshotReadWriter) ReadShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) (io.ReadCloser, error)

func (*NopSnapshotReadWriter) ReadTableKeys 

func (w *NopSnapshotReadWriter) ReadTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) (io.ReadCloser, error)

func (*NopSnapshotReadWriter) WriteFieldKeys 

func (w *NopSnapshotReadWriter) WriteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, wrTo io.WriterTo) error

func (*NopSnapshotReadWriter) WriteShardData 

func (w *NopSnapshotReadWriter) WriteShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, rc io.ReadCloser) error

func (*NopSnapshotReadWriter) WriteTableKeys 

func (w *NopSnapshotReadWriter) WriteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, wrTo io.WriterTo) error

type NopTableKeyReader 

type NopTableKeyReader struct{}

NopTableKeyReader is a no-op implementation of the TableKeyReader interface.

func NewNopTableKeyReader 

func NewNopTableKeyReader() *NopTableKeyReader

func (*NopTableKeyReader) Close 

func (r *NopTableKeyReader) Close() error

func (*NopTableKeyReader) Open 

func (r *NopTableKeyReader) Open() error

func (*NopTableKeyReader) Read 

func (r *NopTableKeyReader) Read() (PartitionKeyMap, error)

type NopWriteLogReader 

type NopWriteLogReader struct{}

NopWriteLogReader is a no-op implementation of the WriteLogReader interface.

func NewNopWriteLogReader 

func NewNopWriteLogReader() *NopWriteLogReader

func (*NopWriteLogReader) FieldKeyReader 

func (w *NopWriteLogReader) FieldKeyReader(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) FieldKeyReader

func (*NopWriteLogReader) ShardReader 

func (w *NopWriteLogReader) ShardReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) ShardReader

func (*NopWriteLogReader) TableKeyReader 

func (w *NopWriteLogReader) TableKeyReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) TableKeyReader

type NopWriteLogWriter 

type NopWriteLogWriter struct{}

NopWriteLogWriter is a no-op implementation of the WriteLogWriter interface.

func NewNopWriteLogWriter 

func NewNopWriteLogWriter() *NopWriteLogWriter

func (*NopWriteLogWriter) CreateFieldKeys 

func (w *NopWriteLogWriter) CreateFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, m map[string]uint64) error

func (*NopWriteLogWriter) CreateTableKeys 

func (w *NopWriteLogWriter) CreateTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, m map[string]uint64) error

func (*NopWriteLogWriter) DeleteFieldKeys 

func (w *NopWriteLogWriter) DeleteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) error

func (*NopWriteLogWriter) DeleteShard 

func (w *NopWriteLogWriter) DeleteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) error

func (*NopWriteLogWriter) DeleteTableKeys 

func (w *NopWriteLogWriter) DeleteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) error

func (*NopWriteLogWriter) WriteShard 

func (w *NopWriteLogWriter) WriteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, msg LogMessage) error

type PartitionKeyMap 

type PartitionKeyMap struct {
	TableKey   dax.TableKey      `json:"table-key"`
	Partition  dax.PartitionNum  `json:"partition"`
	StringToID map[string]uint64 `json:"string-to-id"`
}

type Registrar 

type Registrar interface {
	RegisterNode(ctx context.Context, node *dax.Node) error
	CheckInNode(ctx context.Context, node *dax.Node) error
}

Registrar represents the methods which Computer uses to register itself with MDS.

type RoaringUpdate 

type RoaringUpdate struct {
	Field        string `json:"field"`
	View         string `json:"view"`
	Clear        []byte `json:"clear"`
	Set          []byte `json:"set"`
	ClearRecords bool   `json:"clear-records"`
}

RoaringUpdate is identical to featurebase.RoaringUpdate, but we can't import it due to import cycles. TODO featurebase top level shouldn't import dax stuff... all the types it needs should just be in the top level.

type ShardReader 

type ShardReader interface {
	Open() error
	Read() (LogMessage, error)
	Close() error
}

type SnapshotReadWriter 

type SnapshotReadWriter interface {
	WriteShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, rc io.ReadCloser) error
	ReadShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) (io.ReadCloser, error)

	WriteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, wrTo io.WriterTo) error
	ReadTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) (io.ReadCloser, error)

	WriteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, wrTo io.WriterTo) error
	ReadFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) (io.ReadCloser, error)
}

SnapshotReadWriter provides the interface for all snapshot read and writes in FeatureBase.

type SnapshotService 

type SnapshotService interface {
	Read(bucket string, key string, version int) (io.ReadCloser, error)
	Write(bucket string, key string, version int, rc io.ReadCloser) error
	WriteTo(bucket string, key string, version int, wrTo io.WriterTo) error
}

SnapshotService represents the SnapshotService methods which Computer uses. These are typically implemented by both the Snapshotter client.

type TableKeyReader 

type TableKeyReader interface {
	Open() error
	Read() (PartitionKeyMap, error)
	Close() error
}

type WriteLogReader 

type WriteLogReader interface {
	ShardReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) ShardReader
	TableKeyReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) TableKeyReader
	FieldKeyReader(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) FieldKeyReader
}

WriteLogReader provides the interface for all reads from the write log.

type WriteLogService 

type WriteLogService interface {
	AppendMessage(bucket string, key string, version int, msg []byte) error
	LogReader(bucket string, key string, version int) (io.Reader, io.Closer, error)
	DeleteLog(bucket string, key string, version int) error
}

WriteLogService represents the WriteLogService methods which Computer uses. These are typically implemented by the WriteLogger client.

type WriteLogWriter 

type WriteLogWriter interface {
	// CreateTableKeys sends a map of string key to uint64 ID for the table and
	// partition provided.
	CreateTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, _ map[string]uint64) error

	// DeleteTableKeys deletes all table keys for the table and partition
	// provided.
	DeleteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) error

	// CreateFieldKeys sends a map of string key to uint64 ID for the table and
	// field provided.
	CreateFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, _ map[string]uint64) error

	// DeleteTableKeys deletes all field keys for the table and field provided.
	DeleteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) error

	// WriteShard sends shard data for the table and shard provided.
	WriteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, msg LogMessage) error

	// DeleteShard deletes all data for the table and shard provided.
	DeleteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) error
}

WriteLogWriter provides the interface for all data writes to FeatureBase. After data has been written to the local FeatureBase node, the respective interface method(s) will be called.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.