frames

package module
v0.3.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 18, 2018 License: Apache-2.0 Imports: 18 Imported by: 1

README

frames

Build Status GoDoc License

Server and client library of streaming data from v3io

Components

  • Go server, both gRPC and HTTP protocols are supported
  • Go client
  • Python client
Development

Core is written in Go, we work on development branch and release to `master.

  • To run the Go tests run make test.
  • To run the Python tests run make test-python
Adding/Changing Dependencies
  • If you add Go dependencies run make update-go-deps
  • If you add Python dependencies, updates clients/py/Pipfile and run make update-py-deps
Docker Image
Build
make build-docker
Running
 docker run \
-v /path/to/config.yaml:/etc/framesd.yaml \
quay.io/v3io/frames

LICENSE

Apache 2

Documentation

Overview

Package frames provides an efficient way of moving data from various sources.

The package is composed os a HTTP web server that can serve data from various sources and from clients in Go and in Python.

Index

Constants

View Source
const (
	IgnoreError = pb.ErrorOptions_IGNORE
	FailOnError = pb.ErrorOptions_FAIL
)

Shortcut for fail/ignore

Variables

View Source
var (
	BoolType   = DType(pb.DType_BOOLEAN)
	FloatType  = DType(pb.DType_FLOAT)
	IntType    = DType(pb.DType_INTEGER)
	StringType = DType(pb.DType_STRING)
	TimeType   = DType(pb.DType_TIME)
)

Possible data types

View Source
var (
	// DefaultLogLevel is the default log verbosity
	DefaultLogLevel string
)
View Source
var ZeroTime time.Time

ZeroTime is zero value for time

Functions

func InitBackendDefaults added in v0.3.0

func InitBackendDefaults(cfg *BackendConfig, framesConfig *Config)

InitBackendDefaults initializes default configuration for backend

func NewLogger

func NewLogger(verbose string) (logger.Logger, error)

NewLogger returns a new logger

func SessionFromEnv

func SessionFromEnv() (*pb.Session, error)

SessionFromEnv return a session from V3IO_SESSION environment variable (JSON encoded)

Types

type BackendConfig

type BackendConfig struct {
	Type    string `json:"type"` // v3io, csv, ...
	Name    string `json:"name"`
	Workers int    `json:"workers"`
	// backend specific options
	Options map[string]interface{} `json:"options"`

	// CSV backend
	RootDir string `json:"rootdir,omitempty"`
}

BackendConfig is default backend configuration

type Client

type Client interface {
	// Read reads data from server
	Read(request *ReadRequest) (FrameIterator, error)
	// Write writes data to server
	Write(request *WriteRequest) (FrameAppender, error)
	// Create creates a table
	Create(request *CreateRequest) error
	// Delete deletes data or table
	Delete(request *DeleteRequest) error
	// Exec executes a command on the backend
	Exec(request *ExecRequest) error
}

Client interface

type Column

type Column interface {
	Len() int                                 // Number of elements
	Name() string                             // Column name
	DType() DType                             // Data type (e.g. IntType, FloatType ...)
	Ints() ([]int64, error)                   // Data as []int64
	IntAt(i int) (int64, error)               // Int value at index i
	Floats() ([]float64, error)               // Data as []float64
	FloatAt(i int) (float64, error)           // Float value at index i
	Strings() []string                        // Data as []string
	StringAt(i int) (string, error)           // String value at index i
	Times() ([]time.Time, error)              // Data as []time.Time
	TimeAt(i int) (time.Time, error)          // time.Time value at index i
	Bools() ([]bool, error)                   // Data as []bool
	BoolAt(i int) (bool, error)               // bool value at index i
	Slice(start int, end int) (Column, error) // Slice of data
}

Column is a data column

func NewLabelColumn

func NewLabelColumn(name string, value interface{}, size int) (Column, error)

NewLabelColumn returns a new slabel column

func NewSliceColumn

func NewSliceColumn(name string, data interface{}) (Column, error)

NewSliceColumn returns a new slice column

type Config

type Config struct {
	Log            LogConfig `json:"log"`
	DefaultLimit   int       `json:"limit,omitempty"`
	DefaultTimeout int       `json:"timeout,omitempty"`

	// default V3IO connection details
	WebAPIEndpoint string `json:"webApiEndpoint"`
	Container      string `json:"container"`
	Username       string `json:"username,omitempty"`
	Password       string `json:"password,omitempty"`

	// Number of parallel V3IO worker routines
	Workers int `json:"workers"`

	Backends []*BackendConfig `json:"backends,omitempty"`
}

Config is server configuration

func (*Config) InitDefaults

func (c *Config) InitDefaults() error

InitDefaults initializes the defaults for configuration

func (*Config) Validate

func (c *Config) Validate() error

Validate validates the configuration

type CreateRequest

type CreateRequest = pb.CreateRequest

CreateRequest is a table creation request

type DType

type DType pb.DType

DType is data type

type DataBackend

type DataBackend interface {
	// TODO: Expose name, type, config ... ?
	Read(request *ReadRequest) (FrameIterator, error)
	Write(request *WriteRequest) (FrameAppender, error) // TODO: use Appender for write streaming
	Create(request *CreateRequest) error
	Delete(request *DeleteRequest) error
	Exec(request *ExecRequest) error
}

DataBackend is an interface for read/write on backend

type Decoder

type Decoder struct {
	// contains filtered or unexported fields
}

Decoder is message decoder

func NewDecoder

func NewDecoder(r io.Reader) *Decoder

NewDecoder returns a new Decoder

func (*Decoder) Decode

func (d *Decoder) Decode(msg proto.Message) error

Decode decodes message from d.r

type DeleteRequest

type DeleteRequest = pb.DeleteRequest

DeleteRequest is a deletion request

type Encoder

type Encoder struct {
	// contains filtered or unexported fields
}

Encoder is message encoder

func NewEncoder

func NewEncoder(w io.Writer) *Encoder

NewEncoder returns new Encoder

func (*Encoder) Encode

func (e *Encoder) Encode(msg proto.Message) error

Encode encoders the message to e.w

type ExecRequest

type ExecRequest = pb.ExecRequest

ExecRequest is execution request

type Frame

type Frame interface {
	Labels() map[string]interface{}          // Label set
	Names() []string                         // Column names
	Indices() []Column                       // Index columns
	Len() int                                // Number of rows
	Column(name string) (Column, error)      // Column by name
	Slice(start int, end int) (Frame, error) // Slice of Frame
	IterRows(includeIndex bool) RowIterator  // Iterate over rows
}

Frame is a collection of columns

func NewFrame

func NewFrame(columns []Column, indices []Column, labels map[string]interface{}) (Frame, error)

NewFrame returns a new Frame

func NewFrameFromMap

func NewFrameFromMap(columns map[string]interface{}, indices map[string]interface{}) (Frame, error)

NewFrameFromMap returns a new MapFrame from a map

func NewFrameFromProto

func NewFrameFromProto(msg *pb.Frame) Frame

NewFrameFromProto return a new frame from protobuf message

func NewFrameFromRows

func NewFrameFromRows(rows []map[string]interface{}, indices []string, labels map[string]interface{}) (Frame, error)

NewFrameFromRows creates a new frame from rows

type FrameAppender

type FrameAppender interface {
	Add(frame Frame) error
	WaitForComplete(timeout time.Duration) error
}

FrameAppender appends frames

type FrameIterator

type FrameIterator interface {
	Next() bool
	Err() error
	At() Frame
}

FrameIterator iterates over frames

type JoinStruct

type JoinStruct = pb.JoinStruct

JoinStruct is a join structure

type LogConfig

type LogConfig struct {
	Level string `json:"level,omitempty"`
}

LogConfig is the logging configuration

type Query

type Query struct {
	Table   string
	Columns []string
	Filter  string
	GroupBy string
}

Query is query structure

func ParseSQL

func ParseSQL(sql string) (*Query, error)

ParseSQL parsers SQL query to a Query struct

type ReadRequest

type ReadRequest = pb.ReadRequest

ReadRequest is a read/query request

type RowIterator

type RowIterator interface {
	Next() bool                      // Advance to next row
	Row() map[string]interface{}     // Row as map of name->value
	RowNum() int                     // Current row number
	Indices() map[string]interface{} // MultiIndex as name->value
	Err() error                      // Iteration error
}

RowIterator is an iterator over frame rows

type SchemaField

type SchemaField = pb.SchemaField

SchemaField represents a schema field for Avro record.

type SchemaKey

type SchemaKey = pb.SchemaKey

SchemaKey is a schema key

type Server

type Server interface {
	Start() error
	State() ServerState
	Err() error
}

Server is frames server interface

type ServerBase

type ServerBase struct {
	// contains filtered or unexported fields
}

ServerBase have common functionality for server

func NewServerBase

func NewServerBase() *ServerBase

NewServerBase returns a new server base

func (*ServerBase) Err

func (s *ServerBase) Err() error

Err returns the server error

func (*ServerBase) SetError

func (s *ServerBase) SetError(err error)

SetError sets current error and will change state to ErrorState

func (*ServerBase) SetState

func (s *ServerBase) SetState(state ServerState)

SetState sets the server state

func (*ServerBase) State

func (s *ServerBase) State() ServerState

State return the server state

type ServerState

type ServerState string

ServerState is state of server

const (
	ReadyState   ServerState = "ready"
	RunningState ServerState = "running"
	ErrorState   ServerState = "error"
)

Possible server states

type Session

type Session = pb.Session

Session information

func InitSessionDefaults

func InitSessionDefaults(session *Session, framesConfig *Config) *Session

InitSessionDefaults initializes session defaults

func NewSession

func NewSession(url, container, path, user, password, token string) *Session

NewSession will create a new session. It will populate missing values from the environment. Environment variables have V3IO_ prefix (e.g. V3IO_USER)

type TableSchema

type TableSchema = pb.TableSchema

TableSchema is a table schema

type WriteRequest

type WriteRequest struct {
	Session *Session `msgpack:"session"`
	Backend string   `msgpack:"backend"` // backend name
	Table   string   `msgpack:"table"`   // Table name (path)
	// Data message sent with the write request (in case of a stream multiple messages can follow)
	ImmidiateData Frame `msgpack:"intermidate,omitempty"`
	// Expression template, for update expressions generated from combining columns data with expression
	Expression string `msgpack:"expression,omitempty"`
	// Will we get more message chunks (in a stream), if not we can complete
	HaveMore bool `msgpack:"more"`
}

WriteRequest is request for writing data TODO: Unite with probouf (currenly the protobuf message combines both this and a frame message)

Directories

Path Synopsis
csv
kv
cmd
framesd command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL