collector

type APICollector struct {
	// contains filtered or unexported fields
}

type Article struct {
	ID          string            `json:"id"`
	Title       string            `json:"title"`
	Content     string            `json:"content"`
	Summary     string            `json:"summary"`
	Author      string            `json:"author"`
	URL         string            `json:"url"`
	PublishedAt time.Time         `json:"published_at"`
	Tags        []string          `json:"tags"`
	Source      string            `json:"source"`
	SourceType  string            `json:"source_type"`
	Language    string            `json:"language"`
	Metadata    map[string]string `json:"metadata"`
}

type ArticleFilter struct {
	Keywords  []string  // 关键词过滤
	Authors   []string  // 作者过滤
	Tags      []string  // 标签过滤
	Languages []string  // 语言过滤
	DateFrom  time.Time // 起始日期
	DateTo    time.Time // 结束日期
	MinLength int       // 最小内容长度
	MaxLength int       // 最大内容长度
}

type AtomAuthor struct {
	Name  string `xml:"name"`
	Email string `xml:"email"`
}

type AtomCategory struct {
	Term string `xml:"term,attr"`
}

type AtomContent struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type AtomEntry struct {
	Title     string         `xml:"title"`
	Content   AtomContent    `xml:"content"`
	Summary   string         `xml:"summary"`
	Link      []AtomLink     `xml:"link"`
	ID        string         `xml:"id"`
	Published string         `xml:"published"`
	Updated   string         `xml:"updated"`
	Author    AtomAuthor     `xml:"author"`
	Category  []AtomCategory `xml:"category"`
}

type AtomFeed struct {
	XMLName xml.Name    `xml:"feed"`
	Title   string      `xml:"title"`
	Link    []AtomLink  `xml:"link"`
	Entries []AtomEntry `xml:"entry"`
}

type AtomLink struct {
	Href string `xml:"href,attr"`
	Rel  string `xml:"rel,attr"`
}

type BatchCollector struct {
	// contains filtered or unexported fields
}

type Channel struct {
	Title       string `xml:"title"`
	Description string `xml:"description"`
	Link        string `xml:"link"`
	Items       []Item `xml:"item"`
}

type CollectConfig struct {
	URL         string            `json:"url"`
	Headers     map[string]string `json:"headers,omitempty"`
	Timeout     time.Duration     `json:"timeout,omitempty"`
	MaxArticles int               `json:"max_articles,omitempty"`
	Language    string            `json:"language,omitempty"`
	Tags        []string          `json:"tags,omitempty"`
	Metadata    map[string]string `json:"metadata,omitempty"`
}

type CollectResult struct {
	Articles []Article `json:"articles"`
	Source   string    `json:"source"`
	Error    error     `json:"error,omitempty"`
}

type CollectorManager interface {
	// RegisterCollector 注册采集器
	RegisterCollector(sourceType string, collector DataCollector)

	// CollectAll 并发采集多个数据源
	CollectAll(ctx context.Context, configs []CollectConfig) []CollectResult

	// CollectWithRetry 带重试机制的采集
	CollectWithRetry(ctx context.Context, config CollectConfig, retryConfig RetryConfig) (CollectResult, error)

	// GetCollector 根据类型获取采集器
	GetCollector(sourceType string) (DataCollector, bool)
}

type CollectorManagerImpl struct {
	// contains filtered or unexported fields
}

type DataCollector interface {
	// Collect 采集数据，返回文章列表
	Collect(ctx context.Context, config CollectConfig) (CollectResult, error)

	// GetSourceType 返回采集器类型
	GetSourceType() string

	// Validate 验证配置是否有效
	Validate(config CollectConfig) error
}

type DevToArticle struct {
	ID                 int       `json:"id"`
	Title              string    `json:"title"`
	Description        string    `json:"description"`
	BodyMarkdown       string    `json:"body_markdown"`
	URL                string    `json:"url"`
	PublishedAt        string    `json:"published_at"`
	CreatedAt          string    `json:"created_at"`
	TagList            []string  `json:"tag_list"`
	User               DevToUser `json:"user"`
	Organization       *DevToOrg `json:"organization"`
	ReadingTimeMinutes int       `json:"reading_time_minutes"`
}

type DevToOrg struct {
	Name     string `json:"name"`
	Username string `json:"username"`
}

type DevToUser struct {
	Username string `json:"username"`
	Name     string `json:"name"`
}

type GitHubIssue struct {
	ID        int    `json:"id"`
	Number    int    `json:"number"`
	Title     string `json:"title"`
	Body      string `json:"body"`
	State     string `json:"state"`
	HTMLURL   string `json:"html_url"`
	CreatedAt string `json:"created_at"`
	UpdatedAt string `json:"updated_at"`
	User      struct {
		Login string `json:"login"`
	} `json:"user"`
	Labels []struct {
		Name string `json:"name"`
	} `json:"labels"`
}

type GitHubRepo struct {
	ID              int    `json:"id"`
	Name            string `json:"name"`
	FullName        string `json:"full_name"`
	Description     string `json:"description"`
	HTMLURL         string `json:"html_url"`
	Language        string `json:"language"`
	CreatedAt       string `json:"created_at"`
	UpdatedAt       string `json:"updated_at"`
	PushedAt        string `json:"pushed_at"`
	StargazersCount int    `json:"stargazers_count"`
	ForksCount      int    `json:"forks_count"`
	WatchersCount   int    `json:"watchers_count"`
	OpenIssuesCount int    `json:"open_issues_count"`
	Fork            bool   `json:"fork"`
	Owner           struct {
		Login string `json:"login"`
	} `json:"owner"`
	Topics []string `json:"topics"`
}

type HTMLCollector struct {
	// contains filtered or unexported fields
}

type HTMLSelector struct {
	Title       string `json:"title"`        // 标题选择器
	Content     string `json:"content"`      // 内容选择器
	Summary     string `json:"summary"`      // 摘要选择器
	Author      string `json:"author"`       // 作者选择器
	PublishedAt string `json:"published_at"` // 发布时间选择器
	Tags        string `json:"tags"`         // 标签选择器
	Links       string `json:"links"`        // 链接选择器
}

type Item struct {
	Title       string `xml:"title"`
	Description string `xml:"description"`
	Link        string `xml:"link"`
	GUID        string `xml:"guid"`
	PubDate     string `xml:"pubDate"`
	Author      string `xml:"author"`
	Category    string `xml:"category"`
}

type Link struct {
	URL   string
	Title string
}

type RSSCollector struct {
	// contains filtered or unexported fields
}

type RSSFeed struct {
	XMLName xml.Name `xml:"rss"`
	Channel Channel  `xml:"channel"`
}

type RetryConfig struct {
	MaxRetries int           `json:"max_retries"`
	RetryDelay time.Duration `json:"retry_delay"`
}

type SortBy string