Documentation
¶
Index ¶
- Constants
- Variables
- func IndexField(needle WarehouseField, haystack Schema) int
- func ValueToString(val interface{}, isTime bool) string
- type BaseExportFields
- type BigQuery
- func (bq *BigQuery) ApplyExportSchema(s Schema) error
- func (bq *BigQuery) GetExportTableColumns() []string
- func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema
- func (bq *BigQuery) InitExportTable(s Schema) (bool, error)
- func (bq *BigQuery) LastSyncPoint(_ context.Context) (time.Time, error)
- func (bq *BigQuery) LoadToWarehouse(storageRef string, startTime time.Time) error
- func (bq *BigQuery) SaveSyncPoint(_ context.Context, endTime time.Time) error
- func (bq *BigQuery) ValueToString(val interface{}, isTime bool) string
- type Database
- type GCSStorage
- func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error
- func (g *GCSStorage) GetFilePrefix() string
- func (g *GCSStorage) GetFileReference(name string) string
- func (g *GCSStorage) LastSyncPoint(ctx context.Context) (time.Time, error)
- func (g *GCSStorage) ReadFile(ctx context.Context, name string) (io.Reader, error)
- func (g *GCSStorage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
- func (g *GCSStorage) SaveSyncPoint(ctx context.Context, endTime time.Time) error
- type LocalDisk
- func (w *LocalDisk) DeleteFile(_ context.Context, name string) error
- func (w *LocalDisk) GetFilePrefix() string
- func (w *LocalDisk) GetFileReference(name string) string
- func (w *LocalDisk) LastSyncPoint(ctx context.Context) (time.Time, error)
- func (w *LocalDisk) ReadFile(_ context.Context, name string) (io.Reader, error)
- func (w *LocalDisk) SaveFile(_ context.Context, name string, reader io.Reader) (string, error)
- func (w *LocalDisk) SaveSyncPoint(ctx context.Context, endTime time.Time) error
- type MobileFields
- type Redshift
- func (rs *Redshift) ApplyExportSchema(newSchema Schema) error
- func (rs *Redshift) CopyInData(s3file string) error
- func (rs *Redshift) CreateSyncTable() error
- func (rs *Redshift) DeleteExportRecordsAfter(end time.Time) error
- func (rs *Redshift) DoesTableExist(name string) bool
- func (rs *Redshift) GetExportTableColumns() []string
- func (rs *Redshift) InitExportTable(schema Schema) (bool, error)
- func (rs *Redshift) LastSyncPoint(_ context.Context) (time.Time, error)
- func (rs *Redshift) LoadToWarehouse(s3obj string, _ time.Time) error
- func (rs *Redshift) MakeRedshiftConnection() (*sql.DB, error)
- func (rs *Redshift) RemoveOrphanedRecords(lastSync pq.NullTime) error
- func (rs *Redshift) SaveSyncPoint(_ context.Context, endTime time.Time) error
- func (rs *Redshift) ValueToString(val interface{}, isTime bool) string
- type S3Storage
- func (s *S3Storage) DeleteFile(ctx context.Context, name string) error
- func (s *S3Storage) GetFilePrefix() string
- func (s *S3Storage) GetFileReference(name string) string
- func (s *S3Storage) LastSyncPoint(ctx context.Context) (time.Time, error)
- func (s *S3Storage) ReadFile(ctx context.Context, name string) (io.Reader, error)
- func (s *S3Storage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
- func (s *S3Storage) SaveSyncPoint(ctx context.Context, endTime time.Time) error
- type Schema
- func (s Schema) Equals(other Schema) bool
- func (s Schema) GetFieldForName(col string) WarehouseField
- func (s Schema) GetFullStoryFields() []string
- func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField
- func (s Schema) IsCompatibleWith(other Schema) bool
- func (s Schema) ReconcileWithExisting(colNames []string) Schema
- type Storage
- type SyncViaStorageMixin
- type Syncable
- type ValueToStringFn
- type WarehouseField
Constants ¶
const RFC3339Micro = "2006-01-02T15:04:05.999999Z07:00"
Variables ¶
var (
ErrFileNotFound = errors.New("file not found")
)
Functions ¶
func IndexField ¶
func IndexField(needle WarehouseField, haystack Schema) int
func ValueToString ¶
ValueToString is a common interface method that implementations use to perform value to string conversion
Types ¶
type BaseExportFields ¶
type BaseExportFields struct {
IndvId int64
UserId int64
SessionId int64
PageId int64
UserCreated time.Time
UserAppKey string
UserDisplayName string
UserEmail string
EventStart time.Time
EventType string
EventSubType string
EventCustomName string
EventTargetText string
EventTargetSelector string
EventPageOffset int64
EventSessionOffset int64
EventModFrustrated int64
EventModDead int64
EventModError int64
EventModSuspicious int64
EventVarErrorKind string
EventVarFields string
EventWebSourceFileUrl string
EventFirstInputDelay int64
EventCumulativeLayoutShift float64
SessionStart time.Time
PageName string
PageStart time.Time
PageDuration int64
PageActiveDuration int64
PageUrl string
PageRefererUrl string
PageIp string
PageLatLong string
PageUserAgent string
PageBrowser string
PageBrowserVersion string
PageDevice string
PagePlatform string
PageOperatingSystem string
PageScreenWidth int64
PageScreenHeight int64
PageViewportWidth int64
PageViewportHeight int64
PageNumEvents int32
PageNumDerivedEvents int32
PageNumInfos int64
PageNumWarnings int64
PageNumErrors int64
PageClusterId int64
PageMaxScrollDepthPercent int64
LoadDomContentTime int64
LoadEventTime int64
LoadFirstPaintTime int64
LoadLargestPaintTime int64
ReqUrl string
ReqMethod string
ReqStatus int64
CustomVars string
}
type BigQuery ¶
type BigQuery struct {
// contains filtered or unexported fields
}
func NewBigQuery ¶
func NewBigQuery(c *config.BigQueryConfig) *BigQuery
func (*BigQuery) ApplyExportSchema ¶
func (*BigQuery) GetExportTableColumns ¶
GetExportTableColumns returns a slice of the columns in the existing export table
func (*BigQuery) GetMissingFields ¶
func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema
GetMissingFields returns all fields that are present in the hauserSchema, but not in the bqSchema
func (*BigQuery) LastSyncPoint ¶
func (*BigQuery) LoadToWarehouse ¶
func (*BigQuery) SaveSyncPoint ¶
func (*BigQuery) ValueToString ¶
type Database ¶
type Database interface {
Syncable
LoadToWarehouse(storageRef string, start time.Time) error
ValueToString(val interface{}, isTime bool) string
GetExportTableColumns() []string
// InitExportTable should attempt to create the table in the database. If the table doesn't exist, the provided
// schema should be applied to the table and this function should return `true`, assuming an error didn't occur.
// If the table existed, this should return false to signal that follow-up schema validation is necessary.
InitExportTable(Schema) (bool, error)
// ApplyExportSchema will attempt to update the schema in the database to the provided schema.
// The provided schema must be compatible, or this will fail. Compatible schemas will have existing columns
// in the same order as they are currently ordered in the table and can also add new columns to the end.
ApplyExportSchema(Schema) error
}
type GCSStorage ¶
type GCSStorage struct {
// contains filtered or unexported fields
}
func NewGCSStorage ¶
func NewGCSStorage(conf *config.GCSConfig, gcsClient *storage.Client) *GCSStorage
func (*GCSStorage) DeleteFile ¶
func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error
func (*GCSStorage) GetFilePrefix ¶ added in v1.1.0
func (g *GCSStorage) GetFilePrefix() string
func (*GCSStorage) GetFileReference ¶
func (g *GCSStorage) GetFileReference(name string) string
func (*GCSStorage) LastSyncPoint ¶
func (*GCSStorage) SaveSyncPoint ¶
type LocalDisk ¶
type LocalDisk struct {
// contains filtered or unexported fields
}
func NewLocalDisk ¶
func NewLocalDisk(c *config.LocalConfig) *LocalDisk
func (*LocalDisk) DeleteFile ¶
DeleteFile should do nothing for local disk
func (*LocalDisk) GetFilePrefix ¶ added in v1.1.0
func (*LocalDisk) GetFileReference ¶
func (*LocalDisk) LastSyncPoint ¶
type MobileFields ¶
type MobileFields struct {
AppName string
AppPackageName string
AppDeviceModel string
AppDeviceVendor string
AppVersion string
AppOsVersion string
AppViewName string
EventMobileSourceFile string
}
Mobile Apps fields will not be available for accounts that do not have the feature.
type Redshift ¶
type Redshift struct {
// contains filtered or unexported fields
}
func NewRedshift ¶
func NewRedshift(c *config.RedshiftConfig) *Redshift
func (*Redshift) ApplyExportSchema ¶
func (*Redshift) CopyInData ¶
CopyInData copies data from the given s3File to the export table
func (*Redshift) CreateSyncTable ¶
CreateSyncTable creates a sync table with the hauser sync table schema
func (*Redshift) DeleteExportRecordsAfter ¶
func (*Redshift) DoesTableExist ¶
DoesTableExist checks if a table with a given name exists
func (*Redshift) GetExportTableColumns ¶
GetExportTableColumns returns all the columns of the export table. It opens a connection and calls getTableColumns
func (*Redshift) LastSyncPoint ¶
func (*Redshift) LoadToWarehouse ¶
func (*Redshift) MakeRedshiftConnection ¶
func (*Redshift) RemoveOrphanedRecords ¶
func (*Redshift) SaveSyncPoint ¶
func (*Redshift) ValueToString ¶
type S3Storage ¶
type S3Storage struct {
// contains filtered or unexported fields
}
func NewS3Storage ¶
func (*S3Storage) DeleteFile ¶
func (*S3Storage) GetFilePrefix ¶ added in v1.1.0
func (*S3Storage) GetFileReference ¶
func (*S3Storage) LastSyncPoint ¶
type Schema ¶
type Schema []WarehouseField
func MakeSchema ¶
func MakeSchema(vals ...interface{}) Schema
func (Schema) GetFieldForName ¶
func (s Schema) GetFieldForName(col string) WarehouseField
GetFieldForName takes an existing column name and returns the matching schema field. It performs some conversions between the legacy field names and the new field names.
func (Schema) GetFullStoryFields ¶
func (Schema) GetMissingFieldsFor ¶
func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField
func (Schema) IsCompatibleWith ¶
func (Schema) ReconcileWithExisting ¶
ReconcileWithExisting returns a new schema that is compatible with the provided column names. If exported fields are missing from the column list, they are appended to the end.
type SyncViaStorageMixin ¶
type SyncViaStorageMixin struct {
// contains filtered or unexported fields
}
SyncViaStorageMixin provides a default implementation for the Syncable interface.
func (SyncViaStorageMixin) LastSyncPoint ¶
func (SyncViaStorageMixin) SaveSyncPoint ¶
type ValueToStringFn ¶
type WarehouseField ¶
type WarehouseField struct {
// The name of the field as it exists in the database.
// By default this will match the `FullStoryFieldName`, but may not match for certain fields
// if they've been renamed in the export. (e.g. "PageAgent").
// If the database contains columns that are not part of the FullStory export, this field will still be populated.
DBName string
// The name of the field from FullStory.
// This can be empty if an existing database table contains columns that FullStory does not include
// in the export. This value, if non-empty, is used as part of the `CreateExport` request in the
// `DataExportClient`
FullStoryFieldName string
// FieldType should be used by each database implementation to specify the datatype
// for this column. This is only used when creating or modifying a database's schema.
// If the `FullStoryFieldName` is blank, then this should be nil.
FieldType reflect.Type
}
WarehouseField contains metadata for a field/column in the warehouse.
func (WarehouseField) IsTime ¶
func (f WarehouseField) IsTime() bool