web

type Document struct {
	URL      string            `json:"url"`
	Title    string            `json:"title"`
	Text     string            `json:"text"`
	Markdown string            `json:"markdown"`
	Links    []Link            `json:"links,omitempty"`
	Metadata map[string]string `json:"metadata,omitempty"`
}

type Extractor interface {
	Extract(ctx context.Context, url string, html []byte) (*Document, error)
}

type Link struct {
	Text string `json:"text"`
	URL  string `json:"url"`
}

type Loader struct {
	Reader       Reader
	URLs         []string
	SourcePrefix string
}

type Reader interface {
	Read(ctx context.Context, url string) (*Document, error)
}