sc

type AggFinderVisitor struct {
	Error error
}

type CsvCreatorSettings struct {
	Separator string `json:"separator" yaml:"separator"`
}

type CsvReaderColumnSettings struct {
	SrcColIdx    int    `json:"col_idx,omitempty"`
	SrcColHeader string `json:"col_hdr,omitempty"`
	SrcColFormat string `json:"col_format,omitempty"` // Optional for all except datetime
}

type CsvReaderSettings struct {
	SrcFileHdrLineIdx       int                    `json:"hdr_line_idx" yaml:"hdr_line_idx"`
	SrcFileFirstDataLineIdx int                    `json:"first_data_line_idx,omitempty" yaml:"first_data_line_idx,omitempty"`
	Separator               string                 `json:"separator,omitempty" yaml:"separator,omitempty"`
	ColumnIndexingMode      FileColumnIndexingMode `json:"-"`
}

type CustomProcessorDef interface {
	Deserialize(raw json.RawMessage, customProcSettings json.RawMessage, scriptType ScriptType, caPath string, privateKeys map[string]string) error
	GetFieldRefs() *FieldRefs
	GetUsedInTargetExpressionsFields() *FieldRefs
}

type CustomProcessorDefFactory interface {
	Create(processorType string) (CustomProcessorDef, bool)
}

type DependencyPolicyDef struct {
	EventPriorityOrderString string           `json:"event_priority_order" yaml:"event_priority_order"`
	IsDefault                bool             `json:"is_default" yaml:"is_default"`
	Rules                    []DependencyRule `json:"rules" yaml:"rules"`
	OrderIdxDef              IdxDef
}

type DependencyRule struct {
	Cmd              ReadyToRunNodeCmdType `json:"cmd" yaml:"cmd"`
	RawExpression    string                `json:"expression" yaml:"expression"`
	ParsedExpression ast.Expr
}

type FieldRef struct {
	TableName string
	FieldName string
	FieldType TableFieldType
}

type FieldRefParserFlag uint32

type FieldRefs []FieldRef

type FileColumnIndexingMode string

type FileCreatorDef struct {
	UrlTemplate                   string                 `json:"url_template" yaml:"url_template"`
	RawHaving                     string                 `json:"having,omitempty"`
	Top                           TopDef                 `json:"top,omitempty"`
	Csv                           CsvCreatorSettings     `json:"csv,omitempty"`
	Parquet                       ParquetCreatorSettings `json:"parquet,omitempty"`
	Columns                       []WriteFileColumnDef   `json:"columns" yaml:"columns"`
	Having                        ast.Expr               `json:"-"`
	UsedInHavingFields            FieldRefs              `json:"-"`
	UsedInTargetExpressionsFields FieldRefs              `json:"-"`
	CreatorFileType               int                    `json:"-"`
}

type FileReaderColumnDef struct {
	DefaultValue string                      `json:"col_default_value,omitempty" yaml:"col_default_value,omitempty"` // Optional. If omitted, zero value is used
	Type         TableFieldType              `json:"col_type" yaml:"col_type"`
	Csv          CsvReaderColumnSettings     `json:"csv,omitempty" yaml:"csv,omitempty"`
	Parquet      ParquetReaderColumnSettings `json:"parquet,omitempty" yaml:"parquet,omitempty"`
}

type FileReaderDef struct {
	SrcFileUrls    []string                        `json:"urls" yaml:"urls"`
	Csv            CsvReaderSettings               `json:"csv,omitempty" yaml:"csv,omitempty"`
	Columns        map[string]*FileReaderColumnDef `json:"columns" yaml:"columns"` // Keys are names used in table writer
	ReaderFileType int                             `json:"-"`
}

type IdxCaseSensitivity string

type IdxComponentDef struct {
	FieldName       string
	CaseSensitivity IdxCaseSensitivity
	SortOrder       IdxSortOrder
	StringLen       int64          // For string fields only, default 64
	FieldType       TableFieldType // Populated from tgt_table def
}

type IdxDef struct {
	Uniqueness IdxUniqueness
	Components []IdxComponentDef
}

type IdxDefMap map[string]*IdxDef

type IdxSortOrder string

type IdxUniqueness string

type IndexRef struct {
	TableName string
	IdxName   string
}

type LookupDef struct {
	IndexName                string         `json:"index_name" yaml:"index_name"`
	RawJoinOn                string         `json:"join_on" yaml:"join_on"`
	IsGroup                  bool           `json:"group" yaml:"group"`
	RawFilter                string         `json:"filter" yaml:"filter"`
	LookupJoin               LookupJoinType `json:"join_type" yaml:"join_type"`
	IdxReadBatchSize         int            `json:"idx_read_batch_size" yaml:"idx_read_batch_size"`
	RightLookupReadBatchSize int            `json:"right_lookup_read_batch_size" yaml:"right_lookup_read_batch_size"`

	LeftTableFields    FieldRefs        // In the same order as lookup idx - important
	TableCreator       *TableCreatorDef // Populated when walking through al nodes
	UsedInFilterFields FieldRefs
	Filter             ast.Expr `yaml:"-"`
}

type LookupJoinType string

type NodeRerunPolicy string

type NodeStartPolicy string

type NodeType string

type ParquetCodecType string

type ParquetCreatorSettings struct {
	Codec ParquetCodecType `json:"codec" yaml:"codec"`
}

type ParquetReaderColumnSettings struct {
	SrcColName string `json:"col_name"`
}

type ReadyToRunNodeCmdType string

type ScriptDef struct {
	ScriptNodes           map[string]*ScriptNodeDef  `json:"nodes" yaml:"nodes"`
	RawDependencyPolicies map[string]json.RawMessage `json:"dependency_policies" yaml:"dependency_policies"`
	TableCreatorNodeMap   map[string](*ScriptNodeDef)
	IndexNodeMap          map[string](*ScriptNodeDef)
}

type ScriptInitProblemType int

type ScriptInitResult struct {
	Def         *ScriptDef
	InitProblem ScriptInitProblemType
	Err         error
}

type ScriptNodeDef struct {
	Name                   string          // Get it from the key
	Type                   NodeType        `json:"type" yaml:"type"`
	Desc                   string          `json:"desc" yaml:"desc"`
	StartPolicy            NodeStartPolicy `json:"start_policy" yaml:"start_policy"`
	RerunPolicy            NodeRerunPolicy `json:"rerun_policy,omitempty" yaml:"rerun_policy,omitempty"`
	CustomProcessorType    string          `json:"custom_proc_type,omitempty" yaml:"custom_proc_type,omitempty"`
	HandlerExeType         string          `json:"handler_exe_type,omitempty" yaml:"handler_exe_type,omitempty"`
	MaxBatchProcessingTime int             `json:"max_batch_processing_time,omitempty" yaml:"max_batch_processing_time,omitempty"`

	RawReader   json.RawMessage `json:"r" yaml:"r"` // This depends on tfm type
	TableReader TableReaderDef
	FileReader  FileReaderDef

	Lookup LookupDef `json:"l" yaml:"l"`

	RawProcessorDef json.RawMessage    `json:"p" yaml:"p"` // This depends on tfm type
	CustomProcessor CustomProcessorDef // Also should implement CustomProcessorRunner

	RawWriter            json.RawMessage `json:"w" yaml:"w"` // This depends on tfm type
	DependencyPolicyName string          `json:"dependency_policy" yaml:"dependency_policy"`
	TableCreator         TableCreatorDef
	TableUpdater         TableUpdaterDef
	FileCreator          FileCreatorDef
	DepPolDef            *DependencyPolicyDef
}

type ScriptType string

type TableCreatorDef struct {
	Name                          string                         `json:"name" yaml:"name"`
	CreateProperties              string                         `json:"table_options" yaml:"table_options"`
	RawHaving                     string                         `json:"having,omitempty" yaml:"having,omitempty"`
	Having                        ast.Expr                       `json:"-"`
	UsedInHavingFields            FieldRefs                      `json:"-"`
	UsedInTargetExpressionsFields FieldRefs                      `json:"-"`
	Fields                        map[string]*WriteTableFieldDef `json:"fields,omitempty" yaml:"fields,omitempty"`
	RawIndexes                    map[string]string              `json:"indexes,omitempty" yaml:"indexes,omitempty"`
	Indexes                       IdxDefMap                      `json:"-"`
}

type TableFieldType string

type TableReaderDef struct {
	TableName            string           `json:"table" yaml:"table"`
	ExpectedBatchesTotal int              `json:"expected_batches_total,omitempty" yaml:"expected_batches_total,omitempty"`
	RowsetSize           int              `json:"rowset_size,omitempty" yaml:"rowset_size,omitempty"` // DefaultRowsetSize = 1000, careful with higher values - watch for OOM
	TableCreator         *TableCreatorDef `json:"-"`
}

type TableUpdaterDef struct {
	Fields map[string]*WriteTableFieldDef `json:"fields" yaml:"fields"`
}

type TopDef struct {
	Limit       int    `json:"limit" yaml:"limit"`
	RawOrder    string `json:"order" yaml:"order"`
	OrderIdxDef IdxDef // Not an index really, we just re-use IdxDef infrastructure
}

type WriteCsvColumnSettings struct {
	Format string `json:"format" yaml:"format"`
	Header string `json:"header" yaml:"header"`
}

type WriteFileColumnDef struct {
	RawExpression    string                     `json:"expression" yaml:"expression"`
	Name             string                     `json:"name"` // To be used in Having
	Type             TableFieldType             `json:"type"` // To be checked when checking expressions and to be used in Having
	Csv              WriteCsvColumnSettings     `json:"csv,omitempty"`
	Parquet          WriteParquetColumnSettings `json:"parquet,omitempty"`
	ParsedExpression ast.Expr                   `json:"-"`
	UsedFields       FieldRefs                  `json:"-"`
}

type WriteParquetColumnSettings struct {
	ColumnName string `json:"column_name" yaml:"column_name"`
}

type WriteTableFieldDef struct {
	RawExpression    string         `json:"expression" yaml:"expression"`
	Type             TableFieldType `json:"type" yaml:"type"`
	DefaultValue     string         `json:"default_value,omitempty" yaml:"default_value,omitempty"` // Optional. If omitted, default zero value is used
	ParsedExpression ast.Expr       `json:"-"`
	UsedFields       FieldRefs      `json:"-"`
}