llmflow

type BoundedContainer[E any] struct {
	// contains filtered or unexported fields
}

type Document struct {
	ID       string   `json:"id,omitempty"`
	Text     string   `json:"text,omitempty"`
	Vector   Vector   `json:"vector,omitempty"`
	Metadata Metadata `json:"metadata,omitempty"`

	// Extra data as a JSON string.
	Extra string `json:"extra,omitempty"`
}

type JSONLinesLoader struct {
	orchestrator.TaskHeader

	Input struct {
		ID        string                    `json:"id"`
		Filename  orchestrator.Expr[string] `json:"filename"`
		Content   orchestrator.Expr[string] `json:"content"`
		Pointer   string                    `json:"pointer"`
		BatchSize int                       `json:"batch_size"`
	} `json:"input"`

	Output struct {
		Documents []*Document `json:"documents"`
	}
}

type JSONLinesLoaderBuilder struct {
	// contains filtered or unexported fields
}

type Metadata struct {
	// The source ID of the document.
	//
	// Source/Document has different meanings in different scenarios. For example:
	//
	// 1. Document Splitting
	//   Source => The whole Document
	//   Document => Single Chunk
	//
	// 2. Knowledge Base
	//   Source => The whole Knowledge Base
	//   Document => Single Knowledge Point
	SourceID string `json:"source_id,omitempty"`

	// The user ID. Typically useful in multi-tenant scenario.
	UserID string `json:"user_id,omitempty"`

	// The collection name. Only useful for vector stores that support the
	// concept of Collection (e.g. Milvus, Typesense).
	CollectionID string `json:"collection_id,omitempty"`
}

type Similarity struct {
	*Document `json:",squash"`

	Score float64 `json:"score,omitempty"`
}

type Splitter struct {
	orchestrator.TaskHeader

	Input struct {
		Documents  orchestrator.Expr[[]*Document] `json:"documents"`
		SplitChars []string                       `json:"split_chars"`
		ChunkSize  int                            `json:"chunk_size"`
	} `json:"input"`
}

type SplitterBuilder struct {
	// contains filtered or unexported fields
}

type Template struct {
	orchestrator.TaskHeader

	Input struct {
		Template string                            `json:"template"`
		Args     orchestrator.Expr[map[string]any] `json:"args"`
	} `json:"input"`
}

type TemplateBuilder struct {
	// contains filtered or unexported fields
}

type TextLoader struct {
	orchestrator.TaskHeader

	Input struct {
		ID       string `json:"id"`
		Filename string `json:"filename"`
	} `json:"input"`
}

type TextLoaderBuilder struct {
	// contains filtered or unexported fields
}

type Vector []float64