generator

type FirecrawlClient interface {
	MapWebsite(ctx context.Context, url string, limit int, options FirecrawlOptions) ([]string, error)
	ScrapeURL(ctx context.Context, url string, options FirecrawlOptions) (*ScrapedData, error)
}

type FirecrawlOptions struct {
	OnlyMainContent   bool
	Timeout           int
	Formats           []string
	IncludeSubdomains bool
	IgnoreSitemap     bool
}

type GenerationOptions struct {
	Model            string
	MaxURLs          int
	OutputDir        string
	NoFullText       bool
	Verbose          bool
	BatchSize        int
	MaxWorkers       int
	BatchDelay       time.Duration
	Timeout          time.Duration
	MaxContentLength int
	FirecrawlOptions FirecrawlOptions
}

type GenerationResult struct {
	LLMsTxt        string `json:"llms_txt"`
	LLMsFullTxt    string `json:"llms_full_txt"`
	ProcessedCount int    `json:"processed_count"`
	TotalCount     int    `json:"total_count"`
}

type LLMsTxtGenerator struct {
	// contains filtered or unexported fields
}

type MapResponse struct {
	Success bool     `json:"success"`
	Links   []string `json:"links"`
}

type ProcessedURL struct {
	URL         string `json:"url"`
	Title       string `json:"title"`
	Description string `json:"description"`
	Markdown    string `json:"markdown"`
	Index       int    `json:"index"`
}

type ScrapeResponse struct {
	Success bool `json:"success"`
	Data    struct {
		Markdown string            `json:"markdown"`
		Metadata map[string]string `json:"metadata"`
	} `json:"data"`
}

type ScrapedData struct {
	URL      string            `json:"url"`
	Markdown string            `json:"markdown"`
	Metadata map[string]string `json:"metadata"`
}