Documentation
¶
Index ¶
Constants ¶
const MaxContentLen = 50000
MaxContentLen caps body read at 2x this value via io.LimitReader.
Variables ¶
This section is empty.
Functions ¶
func NormalizeString ¶
NormalizeString trims whitespace and normalizes Unicode
Types ¶
type Cache ¶
type Cache struct {
// contains filtered or unexported fields
}
Cache implements a simple LRU cache with TTL
func (*Cache) Get ¶
func (c *Cache) Get(key string) (*CacheEntry, bool)
Get retrieves an entry from the cache
func (*Cache) Set ¶
func (c *Cache) Set(key string, entry *CacheEntry)
Set adds an entry to the cache
type CacheEntry ¶
type CacheEntry struct {
Bytes int
Code int
CodeText string
ContentType string
Content string
ExpiresAt time.Time
}
CacheEntry stores fetched content with metadata
type Config ¶
type Config struct {
Enabled bool `toml:"enabled"`
AllowDomains []string `toml:"allow_domains"`
BlockDomains []string `toml:"block_domains"`
MaxContentLen int `toml:"max_content_len"` // max markdown output length
}
Config holds web_fetch configuration
func LoadConfigFromEnv ¶
func LoadConfigFromEnv() Config
LoadConfigFromEnv loads configuration from environment variables
func LoadConfigFromToml ¶
LoadConfigFromToml loads configuration from a TOML file
func (Config) MarshalJSON ¶
MarshalJSON implements json.Marshaler for Config
func (*Config) UnmarshalJSON ¶
UnmarshalJSON implements json.Unmarshaler for Config
type DomainPolicy ¶
DomainPolicy controls which domains are allowed/blocked
func DefaultDomainPolicy ¶
func DefaultDomainPolicy() *DomainPolicy
DefaultDomainPolicy returns a default policy that allows all domains
func (*DomainPolicy) IsAllowed ¶
func (p *DomainPolicy) IsAllowed(domain string) bool
IsAllowed checks if a domain is allowed based on the policy
type FetchResult ¶
type FetchResult struct {
URL string `json:"url"`
Code int `json:"code"`
CodeText string `json:"code_text"`
ContentType string `json:"content_type"`
Bytes int `json:"bytes"`
Markdown string `json:"markdown"`
DurationMs int64 `json:"duration_ms"`
}
FetchResult contains the result of a web fetch operation
func FetchURL ¶
func FetchURL(rawURL string, policy *DomainPolicy) (*FetchResult, error)
FetchURL fetches content from a URL and converts it to markdown.
Source Files