Documentation
¶
Index ¶
- type Backend
- type CSVSource
- func NewCSVSource(filePath string, delimiter rune, hasHeader bool) *CSVSource
- func NewCSVSourceFromReadCloser(rc io.ReadCloser, delimiter rune, hasHeader bool) *CSVSource
- func NewHTTPCSVSource(url string, delimiter rune, hasHeader bool, headers map[string]string) *CSVSource
- func NewS3CSVSource(region, bucket, key, endpoint, accessKey, secretKey string, delimiter rune, ...) *CSVSource
- func (s *CSVSource) Ack(ctx context.Context, msg hermod.Message) error
- func (s *CSVSource) Close() error
- func (s *CSVSource) DiscoverDatabases(ctx context.Context) ([]string, error)
- func (s *CSVSource) DiscoverTables(ctx context.Context) ([]string, error)
- func (s *CSVSource) Ping(ctx context.Context) error
- func (s *CSVSource) Read(ctx context.Context) (hermod.Message, error)
- func (s *CSVSource) Sample(ctx context.Context, table string) (hermod.Message, error)
- type Format
- type GenericConfig
- type GenericFileSource
- func (s *GenericFileSource) Ack(ctx context.Context, msg hermod.Message) error
- func (s *GenericFileSource) Close() error
- func (s *GenericFileSource) GetState() map[string]string
- func (s *GenericFileSource) Ping(ctx context.Context) error
- func (s *GenericFileSource) Read(ctx context.Context) (hermod.Message, error)
- func (s *GenericFileSource) Sample(ctx context.Context, table string) (hermod.Message, error)
- func (s *GenericFileSource) SetLogger(l hermod.Logger)
- func (s *GenericFileSource) SetState(state map[string]string)
- type SourceType
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CSVSource ¶
type CSVSource struct {
// contains filtered or unexported fields
}
CSVSource implements the hermod.Source interface for CSV files.
func NewCSVSource ¶
NewCSVSource creates a new CSVSource.
func NewCSVSourceFromReadCloser ¶
func NewCSVSourceFromReadCloser(rc io.ReadCloser, delimiter rune, hasHeader bool) *CSVSource
NewCSVSourceFromReadCloser creates a CSVSource backed by a provided reader (e.g., SFTP file stream).
func NewHTTPCSVSource ¶
func NewHTTPCSVSource(url string, delimiter rune, hasHeader bool, headers map[string]string) *CSVSource
NewHTTPCSVSource creates a new CSVSource for HTTP.
func NewS3CSVSource ¶
func NewS3CSVSource(region, bucket, key, endpoint, accessKey, secretKey string, delimiter rune, hasHeader bool) *CSVSource
NewS3CSVSource creates a new CSVSource for S3.
func (*CSVSource) DiscoverDatabases ¶
func (*CSVSource) DiscoverTables ¶
type GenericConfig ¶
type GenericConfig struct {
Backend Backend
// Common
Pattern string // glob like *.csv (applies to local/ftp; s3 handled via prefix + filter)
Recursive bool
PollInterval time.Duration // how often to rescan when queue is empty (0 = one-shot)
Format Format // raw or csv
// Local
LocalPath string // directory or file path
// HTTP (single file)
URL string
Headers map[string]string
// FTP
FTPAddr string // host:port
FTPUser string
FTPPass string
FTPRootDir string // list from here
FTPTLS bool // reserved; not used for now (plain FTP)
// S3
S3Region string
S3Bucket string
S3Prefix string
S3Endpoint string
S3AccessKey string
S3SecretKey string
}
GenericConfig contains configuration for the generic file source.
type GenericFileSource ¶
type GenericFileSource struct {
// contains filtered or unexported fields
}
GenericFileSource implements a polling file reader producing messages per file or per row (CSV).
func NewGenericFileSource ¶
func NewGenericFileSource(cfg GenericConfig) *GenericFileSource
func (*GenericFileSource) Close ¶
func (s *GenericFileSource) Close() error
func (*GenericFileSource) GetState ¶
func (s *GenericFileSource) GetState() map[string]string
GetState/SetState implement hermod.Stateful to persist watermark.
func (*GenericFileSource) Read ¶
Read implements hermod.Source. It returns a message when available.
func (*GenericFileSource) Sample ¶
Sample reads a single record/file for preview purposes. It is non-destructive: it does not mutate the ingestion queue or the watermark, so calling it from the UI does not skip or consume real data during a subsequent run.
func (*GenericFileSource) SetLogger ¶
func (s *GenericFileSource) SetLogger(l hermod.Logger)
func (*GenericFileSource) SetState ¶
func (s *GenericFileSource) SetState(state map[string]string)
type SourceType ¶
type SourceType string
const ( SourceTypeLocal SourceType = "local" SourceTypeHTTP SourceType = "http" SourceTypeS3 SourceType = "s3" SourceTypeCustom SourceType = "custom" )
Source Files
Click to show internal directories.
Click to hide internal directories.