Documentation
¶
Overview ¶
Package connector turns live data sources into agent-usable knowledge with desensitization as a first-class step. It introspects a source's schema, classifies PII, applies a human-signed MaskingPlan, and feeds the desensitized records into pkg/importflow (RAG + knowledge graph).
Index ¶
- func Desensitized(inner importflow.Source, d *Desensitizer) importflow.Source
- func GeneralizeAge(v string) string
- func GeneralizeValue(kind PiiKind, v string) string
- func MaskValue(kind PiiKind, v string) string
- func NewMCPServer(tb *Toolbox, opts MCPServerOptions) (*mcp.Server, error)
- func NewMySQLSource(dsn string, opts SourceOptions) (importflow.Source, error)
- func NewPostgresSource(dsn string, opts SourceOptions) (importflow.Source, error)
- func Redact(string) string
- func RegisterMCPTools(server *mcp.Server, tb *Toolbox)
- func RunMCPStdio(ctx context.Context, tb *Toolbox, opts MCPServerOptions) error
- func Unmask(ctx context.Context, v Vault, tenant string, tokens []string, kp KeyProvider) (map[string]string, error)
- type ChainClassifier
- type ChangeEvent
- type ChangeOp
- type ChangeSource
- type Checkpoint
- type CheckpointStore
- type Classifier
- type ColumnRule
- type Desensitizer
- type DesensitizerOptions
- type EnvKeyProvider
- type FileKeyProvider
- type KeyProvider
- type LLMClassifier
- type MCPServerOptions
- type MaskAction
- type MaskingPlan
- type MySQLBinlogOptions
- type PiiKind
- type PlanOptions
- type PollingOptions
- type PostgresCDCOptions
- type RuleClassifier
- type SQLiteCheckpointStore
- type SQLiteVault
- type Sensitivity
- type SourceOptions
- type StaticKeyProvider
- type TableCursor
- type TextScanRule
- type TextScanner
- type Toolbox
- type ToolboxOptions
- type Vault
- type Watcher
- type WatcherOptions
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func Desensitized ¶
func Desensitized(inner importflow.Source, d *Desensitizer) importflow.Source
Desensitized returns a Source that applies d to inner. Drops it straight into importflow.New(db).Run(ctx, Desensitized(src, d), plan).
func GeneralizeAge ¶
GeneralizeAge buckets an integer age into a decade band; non-ints pass through.
func GeneralizeValue ¶
GeneralizeValue buckets a value to reduce identifiability. Irreversible.
func MaskValue ¶
MaskValue partially hides a value based on its PII kind, keeping enough for humans to recognize but not enough to identify. Irreversible.
func NewMCPServer ¶ added in v2.23.0
func NewMCPServer(tb *Toolbox, opts MCPServerOptions) (*mcp.Server, error)
NewMCPServer returns a standalone MCP server exposing only the connector tools.
func NewMySQLSource ¶
func NewMySQLSource(dsn string, opts SourceOptions) (importflow.Source, error)
NewMySQLSource connects to MySQL/MariaDB and returns an importflow.Source.
func NewPostgresSource ¶
func NewPostgresSource(dsn string, opts SourceOptions) (importflow.Source, error)
NewPostgresSource connects to Postgres and returns an importflow.Source.
func RegisterMCPTools ¶ added in v2.23.0
RegisterMCPTools adds the connector_* tools (introspect, plan, run, unmask) onto an existing MCP server, so the connector can "ride" another surface — e.g. register it onto the server returned by importflow.NewMCPServer to expose the import and desensitization tools together. Dispatch goes through Toolbox.Call, the single tested code path.
func RunMCPStdio ¶ added in v2.23.0
func RunMCPStdio(ctx context.Context, tb *Toolbox, opts MCPServerOptions) error
RunMCPStdio runs the connector MCP server over stdio until the context is done.
Types ¶
type ChainClassifier ¶
type ChainClassifier []Classifier
ChainClassifier runs classifiers in order; the first non-none result wins, so cheap deterministic rules short-circuit before the LLM is consulted.
func (ChainClassifier) Classify ¶
func (ch ChainClassifier) Classify(ctx context.Context, col importflow.Column, samples []string) (PiiKind, Sensitivity, string)
type ChangeEvent ¶ added in v2.23.0
type ChangeEvent struct {
Op ChangeOp
Table string
Key map[string]string
Row importflow.Record
// Position is the source's resume token for this event (Route B: the LSN /
// binlog position string). Empty for polling sources.
Position string
}
ChangeEvent is one row-level change from a source. Key holds the primary-key column(s) -> raw value(s) and is present for every op (a delete carries only the key). Row holds the full row for Insert/Update and is empty for Delete.
type ChangeOp ¶ added in v2.23.0
type ChangeOp int
ChangeOp is the kind of row-level change captured from a source.
type ChangeSource ¶ added in v2.23.0
type ChangeSource interface {
Changes(ctx context.Context, fn func(ChangeEvent) error) error
Close() error
}
ChangeSource streams row changes. For polling (Route A) Changes returns after one drained batch; for log-based sources (Route B) it blocks and streams until ctx is cancelled. fn returning an error aborts iteration.
func NewMySQLBinlogSource ¶ added in v2.24.0
func NewMySQLBinlogSource(dsn string, opts MySQLBinlogOptions) (ChangeSource, error)
NewMySQLBinlogSource parses the (go-sql-driver) DSN and prepares a binlog reader. It does not start streaming until Changes is called.
func NewPollingChangeSource ¶ added in v2.23.0
func NewPollingChangeSource(driver, dsn string, opts PollingOptions) (ChangeSource, error)
NewPollingChangeSource opens a live SQL source for polling. driver is "postgres"/"pgx" or "mysql".
func NewPostgresCDCSource ¶ added in v2.24.0
func NewPostgresCDCSource(dsn string, opts PostgresCDCOptions) (ChangeSource, error)
NewPostgresCDCSource opens a replication connection and prepares (but does not start) streaming. "replication=database" is appended to the DSN if absent.
type Checkpoint ¶ added in v2.23.0
Checkpoint is a source's resumable position. Cursor carries Route A's per-table watermark (JSON); Position carries Route B's LSN / binlog position. Only one is used per source kind.
type CheckpointStore ¶ added in v2.23.0
type CheckpointStore interface {
Load(ctx context.Context, sourceKey string) (Checkpoint, bool, error)
Save(ctx context.Context, sourceKey string, cp Checkpoint) error
}
CheckpointStore persists and loads a source's Checkpoint by a stable key.
type Classifier ¶
type Classifier interface {
Classify(ctx context.Context, col importflow.Column, samples []string) (PiiKind, Sensitivity, string)
}
Classifier proposes a PII kind + sensitivity for a column from its name, type, and a few SAMPLE values (never the full column).
type ColumnRule ¶
type ColumnRule struct {
Table string `json:"table"`
Column string `json:"column"`
PiiKind PiiKind `json:"pii_kind"`
Sensitivity Sensitivity `json:"sensitivity"`
Action MaskAction `json:"action"`
Reason string `json:"reason,omitempty"` // rule id / LLM note
Source string `json:"source,omitempty"` // "rule" | "llm" | "human"
}
ColumnRule is one column's classification + chosen action.
type Desensitizer ¶
type Desensitizer struct {
// contains filtered or unexported fields
}
Desensitizer applies a signed MaskingPlan to records.
func NewDesensitizer ¶
func NewDesensitizer(plan MaskingPlan, opts DesensitizerOptions) (*Desensitizer, error)
NewDesensitizer validates the plan (must be signed) and returns a Desensitizer.
func (*Desensitizer) Apply ¶
func (d *Desensitizer) Apply(ctx context.Context, r importflow.Record) (importflow.Record, error)
Apply desensitizes one record per the plan. Dropped columns (explicit or unlisted-under-default-deny) are removed entirely; reversible actions write to the vault and emit a token.
type DesensitizerOptions ¶
type DesensitizerOptions struct {
Tenant string
KeyProvider KeyProvider // required when the plan has reversible actions
Vault Vault // required when the plan has reversible actions
TextScanner *TextScanner // defaults to NewTextScanner()
// OnUnlisted is the action applied to a column that the plan does NOT cover
// (no ColumnRule and not a TextScan column). The gate fails CLOSED: the
// default is ActionDrop, so a column the signed plan never classified can
// never leak through — even when the plan is hand-written (e.g. supplied to
// the connector_run tool) and applied to a live source whose real columns
// drifted from the plan. Set it to ActionRedact/ActionKeep only with intent.
OnUnlisted MaskAction
}
DesensitizerOptions configures a Desensitizer.
type EnvKeyProvider ¶
type EnvKeyProvider struct{ Prefix string }
EnvKeyProvider reads a per-tenant key from <Prefix><tenant> (32 raw or 64 hex).
type FileKeyProvider ¶
type FileKeyProvider struct{ Path string }
FileKeyProvider reads a single key from a file (same key for all tenants). The file is either 32 raw bytes or 64 hex chars.
type KeyProvider ¶
KeyProvider supplies a per-tenant 32-byte key. Keys are never logged or stored in the knowledge DB.
type LLMClassifier ¶
type LLMClassifier struct {
Client graphflow.JSONGenerator
}
LLMClassifier asks a model to classify a column from its name + sample values. Used only for columns the rule layer is unsure about (cost + trust boundary).
func (*LLMClassifier) Classify ¶
func (c *LLMClassifier) Classify(ctx context.Context, col importflow.Column, samples []string) (PiiKind, Sensitivity, string)
type MCPServerOptions ¶ added in v2.23.0
type MCPServerOptions struct {
Implementation *mcp.Implementation
Instructions string
Logger *slog.Logger
}
MCPServerOptions configures the connector MCP server wrapper.
type MaskAction ¶
type MaskAction string
MaskAction is what the desensitizer does to a classified column/span.
const ( ActionDrop MaskAction = "drop" // never imported (removed from schema) ActionRedact MaskAction = "redact" // [REDACTED] (irreversible) ActionMask MaskAction = "mask" // partial: 138****1234 ActionHash MaskAction = "hash" // deterministic one-way token (irreversible) ActionPseudonymize MaskAction = "pseudonymize" // reversible via vault ActionGeneralize MaskAction = "generalize" // 34 -> 30-40 (irreversible) ActionKeep MaskAction = "keep" // non-sensitive )
func (MaskAction) Reversible ¶
func (a MaskAction) Reversible() bool
Reversible reports whether an action's original is recoverable from the vault.
type MaskingPlan ¶
type MaskingPlan struct {
Columns []ColumnRule `json:"columns"`
TextScan []TextScanRule `json:"text_scan,omitempty"`
SignedBy string `json:"signed_by,omitempty"`
SignedAt time.Time `json:"signed_at,omitempty"`
}
MaskingPlan is the full, reviewable desensitization decision. Run refuses an unsigned plan (schema-first, data-second).
func BuildMaskingPlan ¶
func BuildMaskingPlan(ctx context.Context, src importflow.Source, cls Classifier, opts PlanOptions) (MaskingPlan, error)
BuildMaskingPlan introspects the source schema (NOT bulk data), classifies each column, and proposes actions. The returned plan is UNSIGNED — the caller must review and Sign() it before Run. Default-deny: a column the classifier is unsure about gets DefaultAction (redact), never keep.
func (MaskingPlan) IsSigned ¶
func (p MaskingPlan) IsSigned() bool
IsSigned reports whether the plan has been approved.
func (MaskingPlan) RuleFor ¶
func (p MaskingPlan) RuleFor(table, column string) (ColumnRule, bool)
RuleFor returns the rule for a table/column, if present.
func (*MaskingPlan) Sign ¶
func (p *MaskingPlan) Sign(by string, at time.Time)
Sign marks the plan approved by a named reviewer.
func (MaskingPlan) TextScanFor ¶
func (p MaskingPlan) TextScanFor(table, column string) bool
TextScanFor reports whether a table/column should be free-text scanned.
type MySQLBinlogOptions ¶ added in v2.24.0
type MySQLBinlogOptions struct {
ServerID uint32 // unique replica id for this client (required; default 1001)
Tables map[string][]string // table -> PK column names (optional; falls back to Table.PKColumns)
}
MySQLBinlogOptions configures the binlog change source.
type PiiKind ¶
type PiiKind string
PiiKind labels what kind of sensitive data a column or span holds.
const ( PiiNone PiiKind = "" PiiName PiiKind = "name" PiiPhone PiiKind = "phone" PiiEmail PiiKind = "email" PiiNationalID PiiKind = "national_id" PiiBankCard PiiKind = "bank_card" PiiAddress PiiKind = "address" PiiDOB PiiKind = "dob" PiiIP PiiKind = "ip" PiiGeo PiiKind = "geo" PiiCustom PiiKind = "custom" )
type PlanOptions ¶
type PlanOptions struct {
// DefaultAction for unclassified columns. Default-deny: ActionRedact.
DefaultAction MaskAction
// ActionFor overrides the action chosen for a given PII kind.
ActionFor map[PiiKind]MaskAction
// ScanTextColumns: columns of type "text" are also marked for free-text scanning.
ScanTextColumns bool
}
PlanOptions tunes plan building.
type PollingOptions ¶ added in v2.23.0
type PollingOptions struct {
Tables []TableCursor
BatchSize int // rows per table per drain; default 500
}
PollingOptions configures a polling change source.
type PostgresCDCOptions ¶ added in v2.24.0
type PostgresCDCOptions struct {
Publication string // pgoutput publication name (required)
Slot string // replication slot name (required)
Tables map[string][]string // table -> primary-key column names (for ChangeEvent.Key)
CreateSlot bool // create a permanent slot if absent (recommended true)
StandbyInterval time.Duration // standby status cadence; default 10s
}
PostgresCDCOptions configures the logical-replication change source.
type RuleClassifier ¶
type RuleClassifier struct{}
RuleClassifier classifies by column-name hints, then by value regex.
func NewRuleClassifier ¶
func NewRuleClassifier() *RuleClassifier
NewRuleClassifier returns a deterministic, dependency-free classifier.
func (*RuleClassifier) Classify ¶
func (c *RuleClassifier) Classify(_ context.Context, col importflow.Column, samples []string) (PiiKind, Sensitivity, string)
Classify implements Classifier. Value-regex evidence overrides a weak name guess.
type SQLiteCheckpointStore ¶ added in v2.23.0
type SQLiteCheckpointStore struct {
// contains filtered or unexported fields
}
SQLiteCheckpointStore stores checkpoints in the knowledge DB, co-located with the data they track.
func NewSQLiteCheckpointStore ¶ added in v2.23.0
func NewSQLiteCheckpointStore(cdb *cortexdb.DB) (*SQLiteCheckpointStore, error)
NewSQLiteCheckpointStore creates the state table on the knowledge DB handle.
func (*SQLiteCheckpointStore) Load ¶ added in v2.23.0
func (s *SQLiteCheckpointStore) Load(ctx context.Context, sourceKey string) (Checkpoint, bool, error)
func (*SQLiteCheckpointStore) Save ¶ added in v2.23.0
func (s *SQLiteCheckpointStore) Save(ctx context.Context, sourceKey string, cp Checkpoint) error
type SQLiteVault ¶
type SQLiteVault struct {
// contains filtered or unexported fields
}
SQLiteVault is a Vault backed by its own SQLite file, separate from the CortexDB knowledge file. Values are AES-256-GCM ciphertext under the tenant key.
func OpenSQLiteVault ¶
func OpenSQLiteVault(path string) (*SQLiteVault, error)
OpenSQLiteVault opens (creating if needed) a vault file.
func (*SQLiteVault) Close ¶
func (v *SQLiteVault) Close() error
type Sensitivity ¶
type Sensitivity int
Sensitivity is an ordered confidentiality level.
const ( Public Sensitivity = iota Internal Confidential Restricted )
type SourceOptions ¶
type SourceOptions struct {
Schema string // DB schema; Postgres default "public"
Tables []string // allow-list; empty = all base tables
SampleSize int // sample rows per table in Schemas(); default 5
RowLimit int // max rows streamed per table; 0 = no limit
}
SourceOptions configures a live SQL source.
type StaticKeyProvider ¶
type StaticKeyProvider []byte
StaticKeyProvider returns a fixed key for every tenant (tests / single-tenant).
type TableCursor ¶ added in v2.23.0
type TableCursor struct {
Table string
CursorColumn string // monotonic column, e.g. updated_at or a sequence
KeyColumns []string // primary key column(s)
}
TableCursor declares how to poll one table for changes.
type TextScanRule ¶
TextScanRule marks a free-text column for in-place PII scanning.
type TextScanner ¶
type TextScanner struct {
// contains filtered or unexported fields
}
TextScanner redacts PII embedded in free text, in place. Layer A is regex (high precision, deterministic). An optional LLM/NER layer is added later.
func NewTextScanner ¶
func NewTextScanner() *TextScanner
NewTextScanner returns the default regex-based scanner.
type Toolbox ¶
type Toolbox struct {
// contains filtered or unexported fields
}
Toolbox exposes the connector as agent-callable tools, designed to be merged into importflow's MCP/toolbox surface.
func NewToolbox ¶
func NewToolbox(db *cortexdb.DB, opts ToolboxOptions) *Toolbox
NewToolbox builds a connector toolbox.
func (*Toolbox) Definitions ¶
func (t *Toolbox) Definitions() []cortexdb.ToolDefinition
Definitions returns the connector tool definitions.
type ToolboxOptions ¶
type ToolboxOptions struct {
Vault Vault
KeyProvider KeyProvider
Tenant string
Classifier Classifier // defaults to NewRuleClassifier()
}
ToolboxOptions wires the connector tools to a vault + classifier.
type Vault ¶
type Vault interface {
Put(ctx context.Context, tenant string, kind PiiKind, original string, kp KeyProvider) (token string, err error)
Resolve(ctx context.Context, tenant string, tokens []string, kp KeyProvider) (map[string]string, error)
Close() error
}
Vault stores reversible original→token mappings outside the knowledge DB.
type Watcher ¶ added in v2.23.0
type Watcher struct {
// contains filtered or unexported fields
}
Watcher consumes ChangeEvents and keeps RAG + KG in sync with the source.
func NewWatcher ¶ added in v2.23.0
func NewWatcher(db *cortexdb.DB, src ChangeSource, opts WatcherOptions) (*Watcher, error)
NewWatcher validates options (incl. the stable-key precondition) and returns a Watcher. It does not start consuming until Run.
func (*Watcher) Run ¶ added in v2.23.0
Run consumes changes until the source's Changes returns (polling: one batch; log-based: until ctx is cancelled), applying each event. For cursor-based sources it seeds the watermark from the checkpoint before draining and saves the advanced watermark after a successful drain.
type WatcherOptions ¶ added in v2.23.0
type WatcherOptions struct {
// SourceKey is the stable checkpoint key for this (source, tableset).
SourceKey string
// Desensitizer applies the signed MaskingPlan. Required.
Desensitizer *Desensitizer
// Mapping routes rows to RAG/KG. Required; every RAG table MUST set IDColumn
// and KG delete targets MUST key on the primary key (validated).
Mapping importflow.MappingPlan
// Checkpoint persists resume position. Required.
Checkpoint CheckpointStore
// Columns gives each table's column list for the one-row import source. If a
// table is absent, columns are inferred from the event row's value keys.
Columns map[string][]importflow.Column
}
WatcherOptions configures a Watcher.