Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
var Drivers = make(map[string]Driver)
Drivers is a registry of drivers
var ErrClosed = errors.New("driver: connection is closed")
ErrClosed indicates the connection is closed
var ErrFileAlreadyExists = errors.New("file already exists")
var ErrNotFound = errors.New("driver: not found")
ErrNotFound indicates the resource wasn't found
var ErrUnsupportedConnector = errors.New("drivers: connector not supported")
ErrUnsupportedConnector is returned from Ingest for unsupported connectors
Functions ¶
Types ¶
type CatalogEntry ¶ added in v0.16.0
type CatalogEntry struct {
Name string
Type ObjectType
Object proto.Message
Path string
CreatedOn time.Time
UpdatedOn time.Time
RefreshedOn time.Time
}
CatalogEntry represents one object in the catalog, such as a source
func (*CatalogEntry) GetMetricsView ¶ added in v0.16.0
func (e *CatalogEntry) GetMetricsView() *runtimev1.MetricsView
func (*CatalogEntry) GetModel ¶ added in v0.16.0
func (e *CatalogEntry) GetModel() *runtimev1.Model
func (*CatalogEntry) GetSource ¶ added in v0.16.0
func (e *CatalogEntry) GetSource() *runtimev1.Source
func (*CatalogEntry) GetTable ¶ added in v0.16.0
func (e *CatalogEntry) GetTable() *runtimev1.Table
type CatalogStore ¶
type CatalogStore interface {
FindEntries(ctx context.Context, instanceID string, t ObjectType) []*CatalogEntry
FindEntry(ctx context.Context, instanceID string, name string) (*CatalogEntry, bool)
CreateEntry(ctx context.Context, instanceID string, entry *CatalogEntry) error
UpdateEntry(ctx context.Context, instanceID string, entry *CatalogEntry) error
DeleteEntry(ctx context.Context, instanceID string, name string) error
}
CatalogStore is implemented by drivers capable of storing catalog info for a specific instance
type Connection ¶
type Connection interface {
// Migrate prepares the connection for use. It will be called before the connection is first used.
// (Not to be confused with migrating artifacts, which is handled by the runtime and tracked in the catalog.)
Migrate(ctx context.Context) error
// MigrationStatus returns the connection's current and desired migration version (if applicable)
MigrationStatus(ctx context.Context) (current int, desired int, err error)
// Close closes the connection
Close() error
// RegistryStore returns a RegistryStore if the driver can serve as such, otherwise returns false.
// The registry is responsible for tracking instances and repos.
RegistryStore() (RegistryStore, bool)
// CatalogStore returns a CatalogStore if the driver can serve as such, otherwise returns false.
// A catalog is used to store state about migrated/deployed objects (such as sources and metrics views).
CatalogStore() (CatalogStore, bool)
// RepoStore returns a RepoStore if the driver can serve as such, otherwise returns false.
// A repo stores file artifacts (either in a folder or virtualized in a database).
RepoStore() (RepoStore, bool)
// OLAPStore returns an OLAPStore if the driver can serve as such, otherwise returns false.
// OLAP stores are where we actually store, transform, and query users' data.
OLAPStore() (OLAPStore, bool)
}
Connection represents a connection to an underlying DB. It should implement one or more of RegistryStore, CatalogStore, RepoStore, and OLAPStore.
type Driver ¶
type Driver interface {
Open(dsn string) (Connection, error)
}
Driver represents an underlying DB
type InformationSchema ¶
type InformationSchema interface {
All(ctx context.Context) ([]*Table, error)
Lookup(ctx context.Context, name string) (*Table, error)
}
InformationSchema contains information about existing tables in an OLAP driver
type Instance ¶
type Instance struct {
// Identifier
ID string
// Driver to connect to for OLAP (options: duckdb, druid)
OLAPDriver string
// DSN for connection to OLAP
OLAPDSN string
// Driver for reading/editing code artifacts (options: file, metastore)
RepoDriver string
// DSN for connecting to repo
RepoDSN string
// EmbedCatalog tells the runtime to store the instance's catalog in its OLAP store instead
// of in the runtime's metadata store. Currently only supported for the duckdb driver.
EmbedCatalog bool `db:"embed_catalog"`
// CreatedOn is when the instance was created
CreatedOn time.Time `db:"created_on"`
// UpdatedOn is when the instance was last updated in the registry
UpdatedOn time.Time `db:"updated_on"`
}
Instance represents a single data project, meaning one OLAP connection, one repo connection, and one catalog connection.
type OLAPStore ¶
type OLAPStore interface {
Dialect() Dialect
Execute(ctx context.Context, stmt *Statement) (*Result, error)
Ingest(ctx context.Context, env *connectors.Env, source *connectors.Source) error
InformationSchema() InformationSchema
}
OLAPStore is implemented by drivers that are capable of storing, transforming and serving analytical queries
type ObjectType ¶ added in v0.16.0
type ObjectType int
Constants representing the kinds of catalog objects
const ( ObjectTypeUnspecified ObjectType = 0 ObjectTypeTable ObjectType = 1 ObjectTypeSource ObjectType = 2 ObjectTypeModel ObjectType = 3 ObjectTypeMetricsView ObjectType = 4 )
type RegistryStore ¶
type RegistryStore interface {
FindInstances(ctx context.Context) ([]*Instance, error)
FindInstance(ctx context.Context, id string) (*Instance, error)
CreateInstance(ctx context.Context, instance *Instance) error
DeleteInstance(ctx context.Context, id string) error
}
RegistryStore is implemented by drivers capable of storing and looking up instances and repos
type RepoObjectStat ¶ added in v0.15.0
type RepoStore ¶
type RepoStore interface {
Driver() string
DSN() string
ListRecursive(ctx context.Context, instID string, glob string) ([]string, error)
Get(ctx context.Context, instID string, path string) (string, error)
Stat(ctx context.Context, instID string, path string) (*RepoObjectStat, error)
Put(ctx context.Context, instID string, path string, reader io.Reader) error
Rename(ctx context.Context, instID string, from string, path string) error
Delete(ctx context.Context, instID string, path string) error
}
RepoStore is implemented by drivers capable of storing code artifacts. It mirrors a file system, but may be virtualized by a database for non-local deployments.
type Result ¶ added in v0.15.0
type Result struct {
*sqlx.Rows
Schema *runtimev1.StructType
}
Result wraps the results of query