documentloaders

type CLICommandLoader ¶ added in v0.2.0

type CLICommandLoader struct {
	Command string
	Args    []string
}

func NewCLICommandLoader(command string, args ...string) *CLICommandLoader

func (l *CLICommandLoader) Load(ctx context.Context) ([]schema.Document, error)

type FileData ¶ added in v0.15.0

type FileData struct {
	Path     string
	Content  string
	FileInfo fs.FileInfo
}

FileData is an in-memory representation of a file to be processed.

type GitLoader ¶

type GitLoader struct {
	// contains filtered or unexported fields
}

GitLoader loads and processes documents from a git repository on the local file system.

func NewGit(path string, registry parsers.ParserRegistry, opts ...GitLoaderOption) (*GitLoader, error)

func (g *GitLoader) Load(ctx context.Context) ([]schema.Document, error)

func (g *GitLoader) LoadAndProcessStream(ctx context.Context, processFn func(ctx context.Context, docs []schema.Document) error) error

type GitLoaderOption ¶ added in v0.2.0

type GitLoaderOption func(*gitLoaderOptions)

func WithBatchSize(size int) GitLoaderOption

func WithExcludeDirs(dirs []string) GitLoaderOption

func WithExcludeExts(exts []string) GitLoaderOption

func WithIncludeExts(exts []string) GitLoaderOption

func WithLogger(logger *slog.Logger) GitLoaderOption

func WithMaxMemoryBuffer(bytes int64) GitLoaderOption

func WithWorkerCount(count int) GitLoaderOption

type Loader ¶

type Loader interface {
	Load(ctx context.Context) ([]schema.Document, error)
	LoadAndProcessStream(ctx context.Context, processFn func(ctx context.Context, docs []schema.Document) error) error
}

type RemoteGitRepoLoader ¶ added in v0.2.0

type RemoteGitRepoLoader struct {
	RepoURL        string
	ParserRegistry parsers.ParserRegistry
	Logger         *slog.Logger
}

func NewRemoteGitRepoLoader(repoURL string, registry parsers.ParserRegistry, logger *slog.Logger) *RemoteGitRepoLoader

func (l *RemoteGitRepoLoader) Load(ctx context.Context) ([]schema.Document, error)