schema

type ColumnSchema struct {
	// SourceName refers to one of 2 things depdending on where the schema is used
	// 1. When the schemas is used by a mapper, SourceName refers to the field name in the raw row data
	// 2. When the schema is used by the JSONL conversion, SourceName refers to the column name in the JSONL
	SourceName string `json:"-"`
	ColumnName string `json:"name,omitempty"`
	// DuckDB type for the column
	Type string `json:"type"`
	// for struct and struct[]
	StructFields []*ColumnSchema `json:"struct_fields,omitempty"`
	// the column description (optional)
	Description string `json:"description,omitempty"`
}

type ColumnType struct {
	// DuckDB type`
	Type string
	// for structs/maps/struct[]
	ChildFields []*ColumnSchema
}

type CommonFields struct {
	// Mandatory fields
	TpID              string    `json:"tp_id"`
	TpSourceType      string    `json:"tp_source_type"`
	TpIngestTimestamp time.Time `json:"tp_ingest_timestamp"`
	TpTimestamp       time.Time `json:"tp_timestamp"`

	// Hive fields
	TpTable     string    `json:"tp_table"`
	TpPartition string    `json:"tp_partition"`
	TpIndex     string    `json:"tp_index"`
	TpDate      time.Time `json:"tp_date" parquet:"type=DATE"`

	// Optional fields
	TpSourceIP       *string `json:"tp_source_ip"`
	TpDestinationIP  *string `json:"tp_destination_ip"`
	TpSourceName     *string `json:"tp_source_name"`
	TpSourceLocation *string `json:"tp_source_location"`

	// Searchable
	TpAkas      []string `json:"tp_akas,omitempty"`
	TpIps       []string `json:"tp_ips,omitempty"`
	TpTags      []string `json:"tp_tags,omitempty"`
	TpDomains   []string `json:"tp_domains,omitempty"`
	TpEmails    []string `json:"tp_emails,omitempty"`
	TpUsernames []string `json:"tp_usernames,omitempty"`
}

type GetColumnDescriptions interface {
	GetColumnDescriptions() map[string]string
}

type GetDescription interface {
	GetDescription() string
}

type Mode string

type ParquetTag struct {
	Name string
	Type string
	Skip bool
}

type RowSchema struct {
	Columns []*ColumnSchema `json:"columns"`
	// should we include ALL source fields in addition to any defined columns, or ONLY include the columns defined
	AutoMapSourceFields bool `json:"automap_source_fields"`
	// should we exclude any source fields from the output (only applicable if automap_source_fields is true)
	ExcludeSourceFields []string `json:"exclude_source_fields"`
	// the table description (optional)
	Description string `json:"description,omitempty"`
}

type SchemaBuilder struct {
	// contains filtered or unexported fields
}

type SchemaMap map[string]*RowSchema

type SourceEnrichment struct {
	// a map of metadata values the source has extracted - perhaps by parsing th artifact path with a grok pattern
	Metadata map[string]string
	// CommonFields - a set of common fields that are added to every row
	CommonFields CommonFields
}