Documentation
¶
Overview ¶
Package frames provides an efficient way of moving data from various sources.
The package is composed os a HTTP web server that can serve data from various sources and from clients in Go and in Python.
Index ¶
- Constants
- Variables
- func InitBackendDefaults(cfg *BackendConfig, framesConfig *Config)
- func NewLogger(verbose string) (logger.Logger, error)
- func SessionFromEnv() (*pb.Session, error)
- type BackendConfig
- type Client
- type Column
- type Config
- type CreateRequest
- type DType
- type DataBackend
- type Decoder
- type DeleteRequest
- type Encoder
- type ExecRequest
- type Frame
- func NewFrame(columns []Column, indices []Column, labels map[string]interface{}) (Frame, error)
- func NewFrameFromMap(columns map[string]interface{}, indices map[string]interface{}) (Frame, error)
- func NewFrameFromProto(msg *pb.Frame) Frame
- func NewFrameFromRows(rows []map[string]interface{}, indices []string, labels map[string]interface{}) (Frame, error)
- type FrameAppender
- type FrameIterator
- type JoinStruct
- type LogConfig
- type Query
- type ReadRequest
- type RowIterator
- type SchemaField
- type SchemaKey
- type Server
- type ServerBase
- type ServerState
- type Session
- type TableSchema
- type WriteRequest
Constants ¶
const ( IgnoreError = pb.ErrorOptions_IGNORE FailOnError = pb.ErrorOptions_FAIL )
Shortcut for fail/ignore
Variables ¶
var ( BoolType = DType(pb.DType_BOOLEAN) FloatType = DType(pb.DType_FLOAT) IntType = DType(pb.DType_INTEGER) StringType = DType(pb.DType_STRING) TimeType = DType(pb.DType_TIME) )
Possible data types
var ( // DefaultLogLevel is the default log verbosity DefaultLogLevel string )
var ZeroTime time.Time
ZeroTime is zero value for time
Functions ¶
func InitBackendDefaults ¶ added in v0.3.0
func InitBackendDefaults(cfg *BackendConfig, framesConfig *Config)
InitBackendDefaults initializes default configuration for backend
func SessionFromEnv ¶
SessionFromEnv return a session from V3IO_SESSION environment variable (JSON encoded)
Types ¶
type BackendConfig ¶
type BackendConfig struct {
Type string `json:"type"` // v3io, csv, ...
Name string `json:"name"`
Workers int `json:"workers"`
// backend specific options
Options map[string]interface{} `json:"options"`
// CSV backend
RootDir string `json:"rootdir,omitempty"`
}
BackendConfig is default backend configuration
type Client ¶
type Client interface {
// Read reads data from server
Read(request *ReadRequest) (FrameIterator, error)
// Write writes data to server
Write(request *WriteRequest) (FrameAppender, error)
// Create creates a table
Create(request *CreateRequest) error
// Delete deletes data or table
Delete(request *DeleteRequest) error
// Exec executes a command on the backend
Exec(request *ExecRequest) error
}
Client interface
type Column ¶
type Column interface {
Len() int // Number of elements
Name() string // Column name
DType() DType // Data type (e.g. IntType, FloatType ...)
Ints() ([]int64, error) // Data as []int64
IntAt(i int) (int64, error) // Int value at index i
Floats() ([]float64, error) // Data as []float64
FloatAt(i int) (float64, error) // Float value at index i
Strings() []string // Data as []string
StringAt(i int) (string, error) // String value at index i
Times() ([]time.Time, error) // Data as []time.Time
TimeAt(i int) (time.Time, error) // time.Time value at index i
Bools() ([]bool, error) // Data as []bool
BoolAt(i int) (bool, error) // bool value at index i
Slice(start int, end int) (Column, error) // Slice of data
}
Column is a data column
func NewLabelColumn ¶
NewLabelColumn returns a new slabel column
func NewSliceColumn ¶
NewSliceColumn returns a new slice column
type Config ¶
type Config struct {
Log LogConfig `json:"log"`
DefaultLimit int `json:"limit,omitempty"`
DefaultTimeout int `json:"timeout,omitempty"`
// default V3IO connection details
WebAPIEndpoint string `json:"webApiEndpoint"`
Container string `json:"container"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
// Number of parallel V3IO worker routines
Workers int `json:"workers"`
Backends []*BackendConfig `json:"backends,omitempty"`
}
Config is server configuration
func (*Config) InitDefaults ¶
InitDefaults initializes the defaults for configuration
type CreateRequest ¶
type CreateRequest = pb.CreateRequest
CreateRequest is a table creation request
type DataBackend ¶
type DataBackend interface {
// TODO: Expose name, type, config ... ?
Read(request *ReadRequest) (FrameIterator, error)
Write(request *WriteRequest) (FrameAppender, error) // TODO: use Appender for write streaming
Create(request *CreateRequest) error
Delete(request *DeleteRequest) error
Exec(request *ExecRequest) error
}
DataBackend is an interface for read/write on backend
type Decoder ¶
type Decoder struct {
// contains filtered or unexported fields
}
Decoder is message decoder
type Encoder ¶
type Encoder struct {
// contains filtered or unexported fields
}
Encoder is message encoder
type Frame ¶
type Frame interface {
Labels() map[string]interface{} // Label set
Names() []string // Column names
Indices() []Column // Index columns
Len() int // Number of rows
Column(name string) (Column, error) // Column by name
Slice(start int, end int) (Frame, error) // Slice of Frame
IterRows(includeIndex bool) RowIterator // Iterate over rows
}
Frame is a collection of columns
func NewFrameFromMap ¶
NewFrameFromMap returns a new MapFrame from a map
func NewFrameFromProto ¶
NewFrameFromProto return a new frame from protobuf message
type FrameAppender ¶
type FrameAppender interface {
Add(frame Frame) error
WaitForComplete(timeout time.Duration) error
}
FrameAppender appends frames
type FrameIterator ¶
FrameIterator iterates over frames
type LogConfig ¶
type LogConfig struct {
Level string `json:"level,omitempty"`
}
LogConfig is the logging configuration
type RowIterator ¶
type RowIterator interface {
Next() bool // Advance to next row
Row() map[string]interface{} // Row as map of name->value
RowNum() int // Current row number
Indices() map[string]interface{} // MultiIndex as name->value
Err() error // Iteration error
}
RowIterator is an iterator over frame rows
type SchemaField ¶
type SchemaField = pb.SchemaField
SchemaField represents a schema field for Avro record.
type Server ¶
type Server interface {
Start() error
State() ServerState
Err() error
}
Server is frames server interface
type ServerBase ¶
type ServerBase struct {
// contains filtered or unexported fields
}
ServerBase have common functionality for server
func (*ServerBase) SetError ¶
func (s *ServerBase) SetError(err error)
SetError sets current error and will change state to ErrorState
func (*ServerBase) SetState ¶
func (s *ServerBase) SetState(state ServerState)
SetState sets the server state
type ServerState ¶
type ServerState string
ServerState is state of server
const ( ReadyState ServerState = "ready" RunningState ServerState = "running" ErrorState ServerState = "error" )
Possible server states
type Session ¶
Session information
func InitSessionDefaults ¶
InitSessionDefaults initializes session defaults
func NewSession ¶
NewSession will create a new session. It will populate missing values from the environment. Environment variables have V3IO_ prefix (e.g. V3IO_USER)
type WriteRequest ¶
type WriteRequest struct {
Session *Session `msgpack:"session"`
Backend string `msgpack:"backend"` // backend name
Table string `msgpack:"table"` // Table name (path)
// Data message sent with the write request (in case of a stream multiple messages can follow)
ImmidiateData Frame `msgpack:"intermidate,omitempty"`
// Expression template, for update expressions generated from combining columns data with expression
Expression string `msgpack:"expression,omitempty"`
// Will we get more message chunks (in a stream), if not we can complete
HaveMore bool `msgpack:"more"`
}
WriteRequest is request for writing data TODO: Unite with probouf (currenly the protobuf message combines both this and a frame message)