Documentation
¶
Overview ¶
Package loader defines the interface for extracting text from various document formats.
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CSVLoader ¶
type CSVLoader struct {
// Separator between columns in output (default: " | ").
Separator string
}
CSVLoader loads CSV files, joining rows into text.
func (*CSVLoader) Describe ¶
func (*CSVLoader) Describe() weave.ComponentInfo
Describe reports the CSV loader.
type DirectoryLoader ¶
type DirectoryLoader struct {
// contains filtered or unexported fields
}
DirectoryLoader recursively walks a directory and loads files using delegate loaders.
func NewDirectoryLoader ¶
func NewDirectoryLoader(loaders ...Loader) *DirectoryLoader
NewDirectoryLoader creates a DirectoryLoader that delegates to the given loaders.
func (*DirectoryLoader) Describe ¶
func (*DirectoryLoader) Describe() weave.ComponentInfo
Describe reports the directory loader. Its Supports never matches a content type, so the Pipeline page lists none for it.
func (*DirectoryLoader) Load ¶
func (l *DirectoryLoader) Load(_ context.Context, _ io.Reader) (*LoadResult, error)
Load is not supported for DirectoryLoader. Use LoadDir.
func (*DirectoryLoader) LoadDir ¶
func (l *DirectoryLoader) LoadDir(ctx context.Context, dirPath string) ([]*LoadResult, error)
LoadDir recursively walks a directory and returns load results for each file.
func (*DirectoryLoader) Supports ¶
func (l *DirectoryLoader) Supports(_ string) bool
Supports always returns false. Use LoadDir directly.
type HTMLLoader ¶
type HTMLLoader struct{}
HTMLLoader extracts text from HTML documents.
func (*HTMLLoader) Describe ¶
func (*HTMLLoader) Describe() weave.ComponentInfo
Describe reports the HTML loader.
func (*HTMLLoader) Load ¶
func (l *HTMLLoader) Load(_ context.Context, reader io.Reader) (*LoadResult, error)
Load reads HTML and returns extracted text content.
func (*HTMLLoader) Supports ¶
func (l *HTMLLoader) Supports(mimeType string) bool
Supports returns true for HTML MIME types.
type JSONLoader ¶
type JSONLoader struct {
// Fields to extract. Empty means extract all string values.
Fields []string
}
JSONLoader loads JSON documents, optionally extracting specific fields.
func NewJSONLoader ¶
func NewJSONLoader(fields ...string) *JSONLoader
NewJSONLoader creates a new JSONLoader.
func (*JSONLoader) Describe ¶
func (*JSONLoader) Describe() weave.ComponentInfo
Describe reports the JSON loader.
func (*JSONLoader) Load ¶
func (l *JSONLoader) Load(_ context.Context, reader io.Reader) (*LoadResult, error)
Load reads JSON and returns extracted text content.
func (*JSONLoader) Supports ¶
func (l *JSONLoader) Supports(mimeType string) bool
Supports returns true for JSON MIME types.
type LoadResult ¶
type LoadResult struct {
// Content is the extracted text content.
Content string `json:"content"`
// Metadata holds format-specific metadata (e.g. title, author, page count).
Metadata map[string]string `json:"metadata,omitempty"`
// MimeType is the detected MIME type of the source document.
MimeType string `json:"mime_type,omitempty"`
}
LoadResult contains the extracted text and metadata from a loaded document.
type Loader ¶
type Loader interface {
// Load reads from the given source and returns the extracted text.
Load(ctx context.Context, reader io.Reader) (*LoadResult, error)
// Supports returns true if this loader can handle the given MIME type.
Supports(mimeType string) bool
}
Loader extracts text content from a document source.
type MarkdownLoader ¶
type MarkdownLoader struct{}
MarkdownLoader strips Markdown formatting and returns plain text.
func NewMarkdownLoader ¶
func NewMarkdownLoader() *MarkdownLoader
NewMarkdownLoader creates a new MarkdownLoader.
func (*MarkdownLoader) Describe ¶
func (*MarkdownLoader) Describe() weave.ComponentInfo
Describe reports the Markdown loader.
func (*MarkdownLoader) Load ¶
func (l *MarkdownLoader) Load(_ context.Context, reader io.Reader) (*LoadResult, error)
Load reads Markdown and returns plain text.
func (*MarkdownLoader) Supports ¶
func (l *MarkdownLoader) Supports(mimeType string) bool
Supports returns true for Markdown MIME types.
type TextLoader ¶
type TextLoader struct{}
TextLoader loads plain text content.
func (*TextLoader) Describe ¶
func (*TextLoader) Describe() weave.ComponentInfo
Describe reports the plain-text loader.
func (*TextLoader) Load ¶
func (l *TextLoader) Load(_ context.Context, reader io.Reader) (*LoadResult, error)
Load reads plain text from the reader.
func (*TextLoader) Supports ¶
func (l *TextLoader) Supports(mimeType string) bool
Supports returns true for text MIME types.
type URLLoader ¶
type URLLoader struct {
// contains filtered or unexported fields
}
URLLoader fetches content from a URL and delegates to an appropriate loader.
func NewURLLoader ¶
NewURLLoader creates a URLLoader that uses the given loader for content extraction.
func NewURLLoaderWithClient ¶
NewURLLoaderWithClient creates a URLLoader with a custom HTTP client.
func (*URLLoader) Describe ¶
func (*URLLoader) Describe() weave.ComponentInfo
Describe reports the URL loader.
func (*URLLoader) Load ¶
func (l *URLLoader) Load(_ context.Context, _ interface{ Read([]byte) (int, error) }) (*LoadResult, error)
Load fetches the URL and delegates content extraction.
Source Files