Documentation
¶
Overview ¶
Package importflow imports external structured data (CSV, MySQL/PG SQL dumps) into CortexDB, building RAG (vector/FTS5) and knowledge-graph (RDF triple) foundations in a single pass. AI assistance (mapping inference, triple extraction, text refinement) is injected via interfaces and always optional; this package imports no LLM SDK.
pkg/importflow/importer.go
pkg/importflow/infer.go
pkg/importflow/refine.go
pkg/importflow/sink_kg.go
pkg/importflow/sink_rag.go
pkg/importflow/source.go
pkg/importflow/source_csv.go
pkg/importflow/source_dump.go
pkg/importflow/toolbox.go
pkg/importflow/types.go
Index ¶
- func NewMCPServer(im *Importer, opts MCPServerOptions) (*mcp.Server, error)
- func RunMCPStdio(ctx context.Context, im *Importer, opts MCPServerOptions) error
- type CSVOptions
- type CSVSource
- type Column
- type Dialect
- type DumpOptions
- type EntityMap
- type Goal
- type Importer
- type KGPlan
- type LLMInferer
- type LLMRefiner
- type MCPServerOptions
- type MappingInferer
- type MappingPlan
- type Option
- type RAGPlan
- type Record
- type RelationMap
- type Report
- type SQLDumpSource
- type Schema
- type Source
- type TablePlan
- type TextExtract
- type TextRefiner
- type Toolbox
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func NewMCPServer ¶ added in v2.20.1
func NewMCPServer(im *Importer, opts MCPServerOptions) (*mcp.Server, error)
NewMCPServer returns an MCP server that exposes the importflow tool surface (importflow_plan, importflow_run) backed by the given Importer.
func RunMCPStdio ¶ added in v2.20.1
func RunMCPStdio(ctx context.Context, im *Importer, opts MCPServerOptions) error
RunMCPStdio runs the importflow MCP server over stdio until the context is done.
Types ¶
type CSVOptions ¶
type CSVOptions struct {
Table string // logical table name; default "csv"
Delimiter rune // default ','
SampleSize int // sample rows in Schemas(); default 5
}
CSVOptions configures a CSVSource.
type CSVSource ¶
type CSVSource struct {
// contains filtered or unexported fields
}
CSVSource is an in-memory Source over a single CSV stream (header required).
func NewCSVSource ¶
func NewCSVSource(r io.Reader, opts CSVOptions) (*CSVSource, error)
NewCSVSource eagerly reads the whole CSV into memory and infers column types.
type Column ¶
type Column struct {
Name string
Type string // "integer","number","text","timestamp","" (unknown)
}
Column is one source column with a best-effort type label.
type DumpOptions ¶
type DumpOptions struct {
Dialect Dialect // default DialectAuto
SampleSize int // sample rows per table in Schemas(); default 5
}
DumpOptions configures a SQLDumpSource.
type EntityMap ¶
type EntityMap struct {
Ref string `json:"ref"` // local handle, e.g. "customer"
Type string `json:"type"` // entity class
IDTmpl string `json:"id_tmpl"` // "{customer_id}"
LabelTmpl string `json:"label_tmpl,omitempty"`
Props []string `json:"props,omitempty"`
}
EntityMap maps columns to one entity per row.
type Importer ¶
type Importer struct {
// contains filtered or unexported fields
}
Importer orchestrates Source -> MappingPlan -> RAG + KG sinks.
func (*Importer) AutoImport ¶
AutoImport runs Plan then Run in one step.
type KGPlan ¶
type KGPlan struct {
Entities []EntityMap `json:"entities,omitempty"`
Relations []RelationMap `json:"relations,omitempty"`
TextExtract []TextExtract `json:"text_extract,omitempty"`
}
KGPlan describes how a row becomes entities and relations.
type LLMInferer ¶
type LLMInferer struct {
Client graphflow.JSONGenerator
}
LLMInferer is the default MappingInferer, backed by a graphflow.JSONGenerator.
func (LLMInferer) InferPlan ¶
func (l LLMInferer) InferPlan(ctx context.Context, schemas []Schema, goal Goal) (MappingPlan, error)
type LLMRefiner ¶
type LLMRefiner struct {
Client graphflow.JSONGenerator
}
LLMRefiner is the default TextRefiner, backed by a graphflow.JSONGenerator.
type MCPServerOptions ¶ added in v2.20.1
type MCPServerOptions struct {
Implementation *mcp.Implementation
Instructions string
Logger *slog.Logger
}
MCPServerOptions configures the importflow MCP server wrapper.
type MappingInferer ¶
type MappingInferer interface {
InferPlan(ctx context.Context, schemas []Schema, goal Goal) (MappingPlan, error)
}
MappingInferer proposes a MappingPlan from table schemas and a goal.
type MappingPlan ¶
MappingPlan declares, per table, how source rows route to RAG and KG.
type Option ¶
type Option func(*Importer)
Option configures an Importer.
func WithBatchSize ¶
WithBatchSize sets the sink batch size (default 500).
func WithExtractor ¶
WithExtractor sets the graphflow extractor used for KGPlan.TextExtract columns.
func WithMappingInferer ¶
func WithMappingInferer(m MappingInferer) Option
WithMappingInferer sets the AI mapping inferer used by Plan/AutoImport.
func WithStrictMode ¶
func WithStrictMode() Option
WithStrictMode aborts the run on the first row error instead of collecting it.
func WithTextRefiner ¶
func WithTextRefiner(r TextRefiner) Option
WithTextRefiner sets the AI text refiner used when RAGPlan.Refine is true.
type RAGPlan ¶
type RAGPlan struct {
Namespace string `json:"namespace,omitempty"`
ContentTmpl string `json:"content_tmpl"` // "{title}\n\n{body}"
IDColumn string `json:"id_column,omitempty"` // default synthesized "table:row"
Metadata []string `json:"metadata,omitempty"` // columns copied into metadata
Refine bool `json:"refine,omitempty"` // run TextRefiner before embedding
}
RAGPlan describes how a row becomes a retrievable text chunk.
type Record ¶
type Record struct {
Table string
Values map[string]string // column name -> string value
Nulls map[string]bool // column name -> is NULL
Row int // 0-based row index within the table
}
Record is one normalized source row.
type RelationMap ¶
type RelationMap struct {
Subject string `json:"subject"`
Predicate string `json:"predicate"`
Object string `json:"object"`
}
RelationMap connects two entity refs with a predicate.
type Report ¶
type Report struct {
RowsRead int
ChunksIndexed int
TriplesCreated int
Skipped int
UnparsedStatements []string
Errors []error
}
Report summarizes an import run. Per repo "no silent caps" rule, dropped / unparsed input is surfaced here rather than discarded silently.
type SQLDumpSource ¶
type SQLDumpSource struct {
// contains filtered or unexported fields
}
SQLDumpSource parses a common subset of MySQL/PG dumps: CREATE TABLE (for column order), INSERT INTO ... VALUES (...), and PG COPY ... \. blocks. Unrecognized statements are recorded in Unparsed() instead of being dropped.
func NewSQLDumpSource ¶
func NewSQLDumpSource(r io.Reader, opts DumpOptions) (*SQLDumpSource, error)
NewSQLDumpSource parses the entire dump eagerly.
func (*SQLDumpSource) Close ¶
func (s *SQLDumpSource) Close() error
func (*SQLDumpSource) Schemas ¶
func (s *SQLDumpSource) Schemas(_ context.Context) ([]Schema, error)
func (*SQLDumpSource) Unparsed ¶
func (s *SQLDumpSource) Unparsed() []string
Unparsed returns statements the parser could not handle.
type Source ¶
type Source interface {
// Schemas returns table schemas with up to a few sample rows each.
Schemas(ctx context.Context) ([]Schema, error)
// Records streams every record; fn returning an error aborts iteration.
Records(ctx context.Context, fn func(Record) error) error
// Close releases any underlying resources.
Close() error
}
Source yields table schemas (for planning) and a stream of records (for import). Implementations should make Schemas() cheap enough to call before Records().
type TablePlan ¶
type TablePlan struct {
Skip bool `json:"skip,omitempty"`
RAG *RAGPlan `json:"rag,omitempty"`
KG *KGPlan `json:"kg,omitempty"`
}
TablePlan is the per-table routing decision.
type TextExtract ¶
TextExtract names a free-text column for AI triple extraction.
type TextRefiner ¶
type TextRefiner interface {
Refine(ctx context.Context, table, column, raw string) (string, error)
}
TextRefiner cleans/summarizes a raw column value before embedding.
type Toolbox ¶
type Toolbox struct {
// contains filtered or unexported fields
}
Toolbox exposes importflow as agent-callable tools.
func NewToolbox ¶
NewToolbox wraps an Importer in a tool surface.
func (*Toolbox) Definitions ¶
func (t *Toolbox) Definitions() []cortexdb.ToolDefinition
Definitions returns the tool definitions (mirrors graphflow's toolbox shape).