Documentation
¶
Overview ¶
Package consolidate provides a pipeline for consolidating raw signals into canonical root causes through embedding, clustering, summarization, and review.
The pipeline consists of five stages:
- Embed: Generate vector embeddings for signals via OmniLLM
- Cluster: Group similar signals by embedding similarity
- Summarize: Generate root cause summaries from clusters via LLM
- Review: Optional human review queue for generated root causes
- Attach: Link new signals to existing root causes incrementally
Curated signals (marked with metadata["curated"]=true) skip the clustering stage and map directly to root causes, preserving their existing aggregation.
Example:
pipeline := consolidate.NewPipeline(
consolidate.WithEmbedder(embedder),
consolidate.WithSummarizer(summarizer),
consolidate.WithSimilarityThreshold(0.85),
)
rootCauses, err := pipeline.Process(ctx, signals)
Index ¶
- Constants
- Variables
- func ClusterSimilarity(a, b Cluster) float64
- func TextFromMetadata(key string) func(signal.Signal) string
- func TextFromSummary() func(signal.Signal) string
- func TextWithEntities() func(signal.Signal) string
- type Cluster
- type ClusterConfig
- type Clusterer
- func (c *Clusterer) AttachToCluster(sig signal.Signal, cluster *Cluster)
- func (c *Clusterer) Cluster(signals []signal.Signal) ([]Cluster, []signal.Signal)
- func (c *Clusterer) FindBestCluster(sig signal.Signal, clusters []Cluster) (*Cluster, float64)
- func (c *Clusterer) MergeClusters(a, b Cluster) Cluster
- type Embedder
- type EmbedderConfig
- type EvidenceLink
- type EvidenceStore
- type IncrementalClusterer
- type LLMClient
- type LLMSummarizer
- type MemoryEvidenceStore
- type MemoryReviewer
- func (r *MemoryReviewer) Approve(ctx context.Context, id string) error
- func (r *MemoryReviewer) ApproveWithDetails(ctx context.Context, id, reviewer, notes string) error
- func (r *MemoryReviewer) Clear()
- func (r *MemoryReviewer) Get(ctx context.Context, id string) (*ReviewItem, error)
- func (r *MemoryReviewer) IsApproved(ctx context.Context, id string) (bool, error)
- func (r *MemoryReviewer) List(ctx context.Context) ([]ReviewItem, error)
- func (r *MemoryReviewer) ListByStatus(ctx context.Context, status string) ([]ReviewItem, error)
- func (r *MemoryReviewer) Pending(ctx context.Context) ([]rootcause.RootCause, error)
- func (r *MemoryReviewer) PendingCount() int
- func (r *MemoryReviewer) Reject(ctx context.Context, id string, reason string) error
- func (r *MemoryReviewer) RejectWithDetails(ctx context.Context, id, reviewer, reason string) error
- func (r *MemoryReviewer) RequiresReview() bool
- func (r *MemoryReviewer) Submit(ctx context.Context, rc rootcause.RootCause) error
- type OmniLLMClient
- type OmniLLMEmbedder
- type Option
- type Pipeline
- type Result
- type ReviewConfig
- type ReviewHook
- type ReviewItem
- type Reviewer
- type RootCauseFilter
- type SkippedSignal
- type Stage
- type Stats
- type Store
- type Summarizer
- type SummarizerConfig
Constants ¶
const ( ReviewPending = "pending" ReviewApproved = "approved" ReviewRejected = "rejected" )
Review status constants.
const DefaultSummarizerPrompt = `` /* 374-byte string literal not displayed */
DefaultSummarizerPrompt is the default system prompt for summarization.
Variables ¶
var ( ErrNoEmbedder = errors.New("embedder not configured") ErrNoSummarizer = errors.New("summarizer not configured") ErrEmptyCluster = errors.New("cluster contains no signals") ErrInvalidConfig = errors.New("invalid pipeline configuration") )
Common errors.
var ( ErrReviewNotFound = errors.New("review not found") ErrAlreadyReviewed = errors.New("root cause already reviewed") ErrReviewerRequired = errors.New("reviewer not configured") )
Common review errors.
Functions ¶
func ClusterSimilarity ¶
ClusterSimilarity computes the similarity between two clusters.
func TextFromMetadata ¶
TextFromMetadata returns a TextFunc that extracts text from a metadata key.
func TextFromSummary ¶
TextFromSummary returns a TextFunc that only uses Summary.
func TextWithEntities ¶
TextWithEntities returns a TextFunc that includes entity names.
Types ¶
type Cluster ¶
type Cluster struct {
// ID is the cluster identifier.
ID string
// Signals are the signals in this cluster.
Signals []signal.Signal
// Centroid is the average embedding of the cluster.
Centroid []float32
// CreatedAt is when the cluster was created.
CreatedAt time.Time
}
Cluster represents a group of similar signals.
type ClusterConfig ¶
type ClusterConfig struct {
// SimilarityThreshold is the minimum cosine similarity for grouping.
// Default is 0.85.
SimilarityThreshold float64
// MinClusterSize is the minimum signals to form a valid cluster.
// Signals in smaller clusters are treated as outliers.
// Default is 1.
MinClusterSize int
// MaxClusterSize limits cluster growth.
// Zero means no limit.
MaxClusterSize int
}
ClusterConfig configures the clustering algorithm.
func DefaultClusterConfig ¶
func DefaultClusterConfig() ClusterConfig
DefaultClusterConfig returns the default clustering configuration.
type Clusterer ¶
type Clusterer struct {
// contains filtered or unexported fields
}
Clusterer groups signals by embedding similarity.
func NewClusterer ¶
func NewClusterer(cfg ClusterConfig) *Clusterer
NewClusterer creates a new signal clusterer.
func (*Clusterer) AttachToCluster ¶
AttachToCluster adds a signal to a cluster and updates the centroid.
func (*Clusterer) Cluster ¶
Cluster groups signals by embedding similarity. Returns clusters and outlier signals that didn't fit any cluster.
func (*Clusterer) FindBestCluster ¶
FindBestCluster returns the cluster most similar to the signal. Returns nil and -1 similarity if no cluster meets the threshold.
func (*Clusterer) MergeClusters ¶
MergeClusters combines two clusters into one.
type Embedder ¶
type Embedder interface {
// Embed generates embeddings for the given signals.
// Returns signals with their Embedding field populated.
Embed(ctx context.Context, signals []signal.Signal) ([]signal.Signal, error)
}
Embedder generates vector embeddings for signals.
type EmbedderConfig ¶
type EmbedderConfig struct {
// Model is the embedding model to use (e.g., "text-embedding-3-small").
Model string
// BatchSize is the maximum signals per embedding request.
// Default is 100.
BatchSize int
// TextFunc extracts the text to embed from a signal.
// Default concatenates Summary and Description.
TextFunc func(signal.Signal) string
}
EmbedderConfig configures an OmniLLM-based embedder.
type EvidenceLink ¶
type EvidenceLink struct {
SignalID string
RootCauseID string
Similarity float64
LinkedAt time.Time
}
EvidenceLink represents the association between a signal and a root cause.
type EvidenceStore ¶
type EvidenceStore interface {
// Link creates an evidence link.
Link(ctx context.Context, link EvidenceLink) error
// GetLinks returns all links for a root cause.
GetLinks(ctx context.Context, rootCauseID string) ([]EvidenceLink, error)
// GetSignalLink returns the link for a specific signal.
GetSignalLink(ctx context.Context, signalID string) (*EvidenceLink, error)
}
EvidenceStore persists signal-to-root-cause evidence links.
type IncrementalClusterer ¶
type IncrementalClusterer struct {
// contains filtered or unexported fields
}
IncrementalClusterer maintains clusters and attaches new signals incrementally.
func NewIncrementalClusterer ¶
func NewIncrementalClusterer(cfg ClusterConfig, existingClusters []Cluster) *IncrementalClusterer
NewIncrementalClusterer creates an incremental clusterer with existing clusters.
func (*IncrementalClusterer) Add ¶
func (ic *IncrementalClusterer) Add(sig signal.Signal) string
Add processes a new signal, either attaching to an existing cluster or creating a new one. Returns the cluster ID the signal was assigned to, or empty if it became an outlier.
func (*IncrementalClusterer) AddBatch ¶
func (ic *IncrementalClusterer) AddBatch(signals []signal.Signal) map[string]string
AddBatch processes multiple signals. Returns a map of signal ID to cluster ID.
func (*IncrementalClusterer) ClusterCount ¶
func (ic *IncrementalClusterer) ClusterCount() int
ClusterCount returns the number of clusters.
func (*IncrementalClusterer) Clusters ¶
func (ic *IncrementalClusterer) Clusters() []Cluster
Clusters returns all current clusters.
type LLMClient ¶
type LLMClient interface {
// Complete generates a completion for the given prompt.
Complete(ctx context.Context, model, systemPrompt, userPrompt string) (string, error)
}
LLMClient is the interface for LLM completion requests.
type LLMSummarizer ¶
type LLMSummarizer struct {
// contains filtered or unexported fields
}
LLMSummarizer implements Summarizer using an LLM.
func NewLLMSummarizer ¶
func NewLLMSummarizer(client LLMClient, cfg SummarizerConfig) *LLMSummarizer
NewLLMSummarizer creates a summarizer backed by an LLM.
type MemoryEvidenceStore ¶
type MemoryEvidenceStore struct {
// contains filtered or unexported fields
}
MemoryEvidenceStore is an in-memory implementation of EvidenceStore.
func NewMemoryEvidenceStore ¶
func NewMemoryEvidenceStore() *MemoryEvidenceStore
NewMemoryEvidenceStore creates an in-memory evidence store.
func (*MemoryEvidenceStore) GetLinks ¶
func (s *MemoryEvidenceStore) GetLinks(ctx context.Context, rootCauseID string) ([]EvidenceLink, error)
GetLinks returns all links for a root cause.
func (*MemoryEvidenceStore) GetSignalLink ¶
func (s *MemoryEvidenceStore) GetSignalLink(ctx context.Context, signalID string) (*EvidenceLink, error)
GetSignalLink returns the link for a specific signal.
func (*MemoryEvidenceStore) Link ¶
func (s *MemoryEvidenceStore) Link(ctx context.Context, link EvidenceLink) error
Link creates an evidence link.
type MemoryReviewer ¶
type MemoryReviewer struct {
// contains filtered or unexported fields
}
MemoryReviewer is an in-memory implementation of the Reviewer interface.
func NewMemoryReviewer ¶
func NewMemoryReviewer(cfg ReviewConfig) *MemoryReviewer
NewMemoryReviewer creates an in-memory review queue.
func (*MemoryReviewer) Approve ¶
func (r *MemoryReviewer) Approve(ctx context.Context, id string) error
Approve marks a root cause as reviewed and approved.
func (*MemoryReviewer) ApproveWithDetails ¶
func (r *MemoryReviewer) ApproveWithDetails(ctx context.Context, id, reviewer, notes string) error
ApproveWithDetails approves with reviewer info and notes.
func (*MemoryReviewer) Clear ¶
func (r *MemoryReviewer) Clear()
Clear removes all review items (for testing).
func (*MemoryReviewer) Get ¶
func (r *MemoryReviewer) Get(ctx context.Context, id string) (*ReviewItem, error)
Get retrieves a review item by root cause ID.
func (*MemoryReviewer) IsApproved ¶
IsApproved checks if a root cause has been approved.
func (*MemoryReviewer) List ¶
func (r *MemoryReviewer) List(ctx context.Context) ([]ReviewItem, error)
List returns all review items.
func (*MemoryReviewer) ListByStatus ¶
func (r *MemoryReviewer) ListByStatus(ctx context.Context, status string) ([]ReviewItem, error)
ListByStatus returns review items with the specified status.
func (*MemoryReviewer) PendingCount ¶
func (r *MemoryReviewer) PendingCount() int
PendingCount returns the number of items awaiting review.
func (*MemoryReviewer) RejectWithDetails ¶
func (r *MemoryReviewer) RejectWithDetails(ctx context.Context, id, reviewer, reason string) error
RejectWithDetails rejects with reviewer info.
func (*MemoryReviewer) RequiresReview ¶
func (r *MemoryReviewer) RequiresReview() bool
RequiresReview returns true if root causes must be approved before use.
type OmniLLMClient ¶
type OmniLLMClient interface {
// Embed generates embeddings for the given texts.
Embed(ctx context.Context, model string, texts []string) ([][]float32, error)
}
OmniLLMClient is the interface for embedding requests. This matches the omnillm.Client embedding API.
type OmniLLMEmbedder ¶
type OmniLLMEmbedder struct {
// contains filtered or unexported fields
}
OmniLLMEmbedder implements Embedder using OmniLLM.
func NewOmniLLMEmbedder ¶
func NewOmniLLMEmbedder(client OmniLLMClient, cfg EmbedderConfig) *OmniLLMEmbedder
NewOmniLLMEmbedder creates an embedder backed by OmniLLM.
type Option ¶
type Option func(*Pipeline)
Option configures a Pipeline.
func WithMinClusterSize ¶
WithMinClusterSize sets the minimum signals required to form a cluster. Default is 1.
func WithRequireReview ¶
WithRequireReview enables mandatory human review for generated root causes.
func WithReviewer ¶
WithReviewer sets the review queue handler.
func WithSimilarityThreshold ¶
WithSimilarityThreshold sets the clustering similarity threshold. Signals with embedding similarity >= threshold are grouped together. Default is 0.85.
func WithSummarizer ¶
func WithSummarizer(s Summarizer) Option
WithSummarizer sets the summarization provider.
type Pipeline ¶
type Pipeline struct {
// contains filtered or unexported fields
}
Pipeline orchestrates signal consolidation.
func NewPipeline ¶
NewPipeline creates a new consolidation pipeline.
type Result ¶
type Result struct {
// RootCauses are the generated or updated root causes.
RootCauses []rootcause.RootCause
// Clusters are the signal clusters formed.
Clusters []Cluster
// Attached are signals linked to existing root causes.
Attached map[string]string // signal ID -> root cause ID
// Skipped are signals that couldn't be processed.
Skipped []SkippedSignal
// Stats holds processing statistics.
Stats Stats
}
Result holds the output of pipeline processing.
type ReviewConfig ¶
type ReviewConfig struct {
// RequireReview blocks root causes until approved.
// If false, unreviewed root causes are flagged but not blocked.
RequireReview bool
// OnSubmit is called when a root cause is submitted for review.
OnSubmit ReviewHook
// OnApprove is called when a root cause is approved.
OnApprove ReviewHook
// OnReject is called when a root cause is rejected.
OnReject ReviewHook
// AutoApproveThreshold auto-approves if signal count exceeds this.
// Zero disables auto-approve.
AutoApproveThreshold int
}
ReviewConfig configures the review system.
type ReviewHook ¶
type ReviewHook func(ctx context.Context, item ReviewItem) error
ReviewHook is called when review events occur.
type ReviewItem ¶
type ReviewItem struct {
RootCause rootcause.RootCause
SubmittedAt time.Time
ReviewedAt *time.Time
ReviewedBy string
Status string
Notes string
}
ReviewItem represents a root cause pending review.
type Reviewer ¶
type Reviewer interface {
// Submit queues a root cause for review.
Submit(ctx context.Context, rc rootcause.RootCause) error
// Pending returns root causes awaiting review.
Pending(ctx context.Context) ([]rootcause.RootCause, error)
// Approve marks a root cause as reviewed and approved.
Approve(ctx context.Context, id string) error
// Reject marks a root cause as reviewed and rejected.
Reject(ctx context.Context, id string, reason string) error
}
Reviewer handles human review of generated root causes.
type RootCauseFilter ¶
type RootCauseFilter struct {
// Status filters by root cause status.
Status []rootcause.Status
// Since filters root causes created after this time.
Since time.Time
// Domain filters by domain name.
Domain string
// Limit is the maximum number of results.
Limit int
}
RootCauseFilter specifies criteria for listing root causes.
type SkippedSignal ¶
SkippedSignal records a signal that was skipped during processing.
type Stats ¶
type Stats struct {
// SignalsProcessed is the total signals processed.
SignalsProcessed int
// SignalsEmbedded is the count embedded successfully.
SignalsEmbedded int
// SignalsClustered is the count assigned to clusters.
SignalsClustered int
// SignalsAttached is the count attached to existing root causes.
SignalsAttached int
// ClustersFormed is the number of new clusters created.
ClustersFormed int
// RootCausesGenerated is the number of new root causes.
RootCausesGenerated int
// Duration is the total processing time.
Duration time.Duration
}
Stats holds pipeline processing statistics.
type Store ¶
type Store interface {
// SaveRootCause persists a root cause.
SaveRootCause(ctx context.Context, rc rootcause.RootCause) error
// GetRootCause retrieves a root cause by ID.
GetRootCause(ctx context.Context, id string) (rootcause.RootCause, error)
// ListRootCauses returns all root causes matching the filter.
ListRootCauses(ctx context.Context, filter RootCauseFilter) ([]rootcause.RootCause, error)
// LinkSignal associates a signal with a root cause.
LinkSignal(ctx context.Context, signalID, rootCauseID string) error
// GetLinkedSignals returns signal IDs linked to a root cause.
GetLinkedSignals(ctx context.Context, rootCauseID string) ([]string, error)
}
Store persists root causes and signal-to-root-cause links.
type Summarizer ¶
type Summarizer interface {
// Summarize generates a root cause from a cluster of signals.
Summarize(ctx context.Context, cluster Cluster) (rootcause.RootCause, error)
}
Summarizer generates root cause summaries from signal clusters.
type SummarizerConfig ¶
type SummarizerConfig struct {
// Model is the LLM model to use.
Model string
// SystemPrompt customizes the summarization behavior.
// If empty, uses DefaultSummarizerPrompt.
SystemPrompt string
// MaxSignals limits how many signals to include in context.
// Default is 20.
MaxSignals int
// IncludeMetadata includes signal metadata in the prompt.
IncludeMetadata bool
}
SummarizerConfig configures the LLM-based summarizer.