catalogimport

type Catalog interface {
	GetDataset(ctx context.Context, ref catalog.Ref) (*catalog.Dataset, error)
	GetDefinition(ctx context.Context, ref catalog.Ref) (*catalog.Definition, error)
	PutDefinition(ctx context.Context, in catalog.DefinitionInput) (*catalog.Definition, error)
	History(ctx context.Context, ref catalog.Ref, name catalog.AspectName) ([]catalog.Change, error)
	WriteAspects(ctx context.Context, edits []catalog.AspectEdit, by string) ([]catalog.Change, error)
	AddEdges(ctx context.Context, edges []catalog.Edge) error
	DatasetsByPath(ctx context.Context, connectionKind, path string) ([]string, error)
}

type Client struct {
	// contains filtered or unexported fields
}

type ContextDocument struct {
	ID       string
	Title    string
	Content  string
	Category string
}

type DataHub struct {
	// contains filtered or unexported fields
}

type DatasetCounts struct {
	// Read is every dataset record DataHub listed, removed ones included.
	Read int `json:"read"`
	// Imported is the catalog datasets the import wrote curation onto.
	Imported int `json:"imported"`
	// FromRemoved is the catalog datasets that took curation from a record
	// DataHub has marked removed.
	FromRemoved int `json:"from_removed"`
	Skipped     int `json:"skipped"`
}

type DatasetHead struct {
	URN     string
	Removed bool
}

type DatasetRecord struct {
	URN     string
	Removed bool
	Aspects catalog.Aspects
	// Columns are the curation of each top-level field, by field name.
	Columns map[string]catalog.Aspects
	// Upstreams are the URNs of the datasets DataHub records this one is
	// derived from.
	Upstreams []string
}

type Definition struct {
	URN  string
	Kind catalog.Kind
	Name string
	// Parent is the URN of the glossary node a term or node sits under.
	Parent  string
	Aspects catalog.Aspects
}

type GraphQL interface {
	Execute(ctx context.Context, query string, variables map[string]any, result any) error
	GetContextDocuments(ctx context.Context, urn string) ([]ContextDocument, error)
}

type Importer struct {
	// contains filtered or unexported fields
}

type Mapping interface {
	Map(ctx context.Context) (map[string]string, error)
	Record(ctx context.Context, urn, reference string) error
}

type Options struct {
	// Platform is the DataHub platform the deployment's Trino datasets were
	// recorded under when it is not "trino".
	Platform string
	// CatalogMapping maps a query-engine catalog name to the name DataHub
	// knows it by; the import reads it backwards.
	CatalogMapping map[string]string
}

type Report struct {
	// Definitions counts the definitions imported, by kind.
	Definitions map[string]int `json:"definitions"`
	Datasets    DatasetCounts  `json:"datasets"`
	// AspectsWritten counts the aspects the import wrote; AspectsUnchanged
	// those that already held DataHub's value; AspectsKept those it left
	// alone because something other than the import wrote them in the
	// platform catalog.
	AspectsWritten   int `json:"aspects_written"`
	AspectsUnchanged int `json:"aspects_unchanged"`
	AspectsKept      int `json:"aspects_kept"`
	LineageEdges     int `json:"lineage_edges"`
	// Skipped lists what was not imported and why, at most maxListed of
	// them; SkippedByReason counts every one.
	Skipped         []Skip         `json:"skipped"`
	SkippedByReason map[string]int `json:"skipped_by_reason"`
	// Rewrite is what the rewrite of stored references did.
	Rewrite *RewriteReport `json:"rewrite,omitempty"`
}

type Resolver func(urn string) string

type Rewrite interface {
	Rewrite(ctx context.Context, resolve Resolver, report *Report) error
}

type RewriteReport struct {
	// Rewritten counts the rows changed, by table and column.
	Rewritten map[string]int `json:"rewritten"`
	// Unmapped counts the references left as they were, because the import
	// has no catalog entity for them; each is listed in the report's skips
	// with where it was found.
	Unmapped int `json:"unmapped"`
	// IndexRowsDropped counts the rows of the dataset search index keyed by a
	// DataHub URN. The index reads the platform catalog; it re-reads the
	// datasets those rows named under their catalog references.
	IndexRowsDropped int `json:"index_rows_dropped"`
}

type Rewriter struct {
	// contains filtered or unexported fields
}

type Run struct {
	ID         string     `json:"id"`
	SourceURL  string     `json:"source_url"`
	StartedBy  string     `json:"started_by"`
	StartedAt  time.Time  `json:"started_at"`
	FinishedAt *time.Time `json:"finished_at,omitempty"`
	// Running is true while the run holds the import's lock; a run with no
	// finished_at that does not was cut off when its replica stopped.
	Running bool    `json:"running"`
	Report  *Report `json:"report,omitempty"`
	Error   string  `json:"error,omitempty"`
}

type Runner struct {
	// contains filtered or unexported fields
}

type Skip struct {
	URN    string `json:"urn"`
	Reason string `json:"reason"`
	// Where names the platform table and column a rewrite skip was found in.
	Where string `json:"where,omitempty"`
}

type Source interface {
	DatasetHeads(ctx context.Context) ([]DatasetHead, error)
	Dataset(ctx context.Context, urn string) (*DatasetRecord, error)
	Definitions(ctx context.Context) ([]Definition, error)
}

type SourceConfig struct {
	URL   string
	Token string
}

type SourceFactory func(url, token string) (Source, error)

type Store struct {
	// contains filtered or unexported fields
}