define

package
v0.3.9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 26, 2023 License: MIT Imports: 1 Imported by: 0

Documentation

Index

Constants

View Source
const (
	CHARSET_GBK     = "gbk"
	CHARSET_GB2312  = "gb2312"
	CHARSET_GB18030 = "gb18030"
	CHARSET_BIG5    = "big5"

	CHARSET_UTF8 = "utf-8"
)

RSS Can supported page charsets

View Source
const (
	DEFAULT_DEBUG_MODE      = false
	DEFAULT_DEBUG_LEVEL     = "info" // debug logger level: `debug`, `info`, `warn`, `error`
	DEFAULT_USER_AGENT      = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
	DEFAULT_REQUEST_TIMEOUT = 5 // seconds, fetch remote data timeout
	DEFAULT_SERVER_TIMEOUT  = 8 // seconds, web server request timeout
	DEFAULT_RULES_DIRECTORY = "./rules"
	DEFAULT_HTTP_HOST       = "0.0.0.0"
	DEFAULT_HTTP_PORT       = 8080
	DEFAULT_HTTP_FEED_PATH  = "/feed"
)
View Source
const (
	DEFAULT_REDIS        = true
	DEFAULT_REDIS_SERVER = "127.0.0.1:6379"
	DEFAULT_REDIS_PASS   = ""
	DEFAULT_REDIS_DB     = 0
)
View Source
const (
	DEFAULT_IN_MEMORY_CACHE            = true
	DEFAULT_IN_MEMORY_CACHE_EXPIRATION = 10 * 60 //seconds, 10min
	DEFAULT_IN_MEMORY_CACHE_STORE_NAME = "memory_cache"
)
View Source
const (
	DEFAULT_HEADLESS_SERVER         = "127.0.0.1:9222"
	DEFAULT_PROXY_ADDRESS           = ""
	DEFAULT_JS_EXECUTE_TIMEOUT      = 200 // milliseconds
	DEFAULT_JS_EXECUTE_THORTTLING   = 2   // seconds
	DEFAULT_HEADLESS_SLOW_MOTION    = 2   // seconds
	DEFAULT_HEADLESS_EXCUTE_TIMEOUT = 5   // Second
)
View Source
const (
	ERROR_STATUS_NULL                  string = "一切正常"
	ERROR_STATUS_NETWORK               string = "网络故障"
	ERROR_STATUS_INIT_NETWORK_FAILED   string = "初始化网络组件错误"
	ERROR_STATUS_API_NOT_READY         string = "目标接口未就绪"
	ERROR_STATUS_DECODE_CAHRSET_FAILED string = "解析内容编码出错"
	ERROR_STATUS_PARSE_CONTENT_FAILED  string = "解析文档出错"
)
View Source
const (
	FEED_TYPE_RSS  = "rss"
	FEED_TYPE_ATOM = "atom"
	FEED_TYPE_JSON = "json"
)
View Source
const (
	FEED_MIME_TYPE_RSS     = "application/rss+xml"
	FEED_MIME_TYPE_ATOM    = "application/atom+xml"
	FEED_MIME_TYPE_JSON    = "application/feed+json"
	FEED_MIME_TYPE_DEFAULT = "text/plain"
)
View Source
const (
	DEFAULT_PARSE_MODE = PARSE_MODE_SSR // Use `ssr` as default and fallback for document parsing
	PARSE_MODE_SSR     = "ssr"
	PARSE_MODE_CSR     = "csr"
	PARSE_MODE_MIX     = "mix"
)

Parser mode types

View Source
const (
	ConfigHookReadLink = "readlink"
)
View Source
const (
	// Use UTF-8 encoding as default and fallback for document detection
	DEFAULT_DOCUMENT_CHARSET = CHARSET_UTF8
)

Variables

View Source
var (
	DEBUG_MODE      = false
	DEBUG_LEVEL     = "info"
	USER_AGENT      = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
	REQUEST_TIMEOUT = 45 // seconds, fetch remote data timeout
	SERVER_TIMEOUT  = 50 // seconds, web server request timeout
	RULES_DIRECTORY = "./rules"
	HTTP_HOST       = "0.0.0.0"
	HTTP_PORT       = 8080
	HTTP_FEED_PATH  = "/feed"

	REDIS        = true
	REDIS_SERVER = "127.0.0.1:6379"
	REDIS_PASS   = ""
	REDIS_DB     = 0

	IN_MEMORY_CACHE      = true
	IN_MEMORY_EXPIRATION = 10 * 60 //seconds, 10min

	HEADLESS_SERVER         = "127.0.0.1:9222"
	PROXY_SERVER            = ""
	JS_EXECUTE_TIMEOUT      = 200 // milliseconds
	HEADLESS_SLOW_MOTION    = 2   // seconds
	HEADLESS_EXCUTE_TIMEOUT = 10  // Second
)
View Source
var (
	CSR_DEBUG              = false
	CSR_INCOGNITO          = false
	CSR_IGNORE_CERT_ERRORS = true
)

CSR features switch

Functions

This section is empty.

Types

type BodyParsed

type BodyParsed struct {
	Code   ErrorCode  `json:"code"`
	Status string     `json:"status"`
	Date   time.Time  `json:"date"`
	Body   []InfoItem `json:"data,omitempty"`
}

func MixupBodyParsed

func MixupBodyParsed(code ErrorCode, status string, date time.Time, data []InfoItem) (result BodyParsed)

type ConfigHook

type ConfigHook struct {
	Action string `json:"action"`
	Object string `json:"object"`
	URL    string `json:"URL"`
}

type ErrorCode

type ErrorCode int
const (
	ERROR_CODE_NULL                  ErrorCode = 0
	ERROR_CODE_NETWORK               ErrorCode = 1
	ERROR_CODE_INIT_NETWORK_FAILED   ErrorCode = 2
	ERROR_CODE_API_NOT_READY         ErrorCode = 3
	ERROR_CODE_DECODE_CAHRSET_FAILED ErrorCode = 4
	ERROR_CODE_PARSE_CONTENT_FAILED  ErrorCode = 5
)

type InfoItem

type InfoItem struct {
	ID          string `json:"id,omitempty"`
	Title       string `json:"title"`
	Link        string `json:"link"`
	Date        string `json:"date"`
	Author      string `json:"author,omitempty"`
	Category    string `json:"category,omitempty"`
	Description string `json:"description,omitempty"`
	Content     string `json:"content,omitempty"`
}

type JavaScriptConfig

type JavaScriptConfig struct {
	Name         string  `json:"name"`
	URL          string  `json:"URL"`
	Mode         string  `json:"Mode"`
	File         string  //private field
	Charset      string  `json:"Charset"`
	Expire       int     `json:"Expire"`
	Headless     string  `json:"Headless"`
	Timeout      int     `json:"Timeout"`
	Proxy        string  `json:"Proxy"`
	DisableCache bool    `json:"DisableCache"`
	IdByRegexp   string  `json:"IdByRegexp"`
	IdByProp     PROP_ID `json:"IdByProp"`
	Cookies      string  `json:"Cookies"`

	ListContainer string `json:"ListContainer"`
	Title         string `json:"Title"`
	Author        string `json:"Author"`
	Link          string `json:"Link"`

	DateTime     string     `json:"DateTime"`
	DateTimeHook ConfigHook `json:"DateTimeHook"`

	Category     string     `json:"Category"`
	CategoryHook ConfigHook `json:"CategoryHook"`

	Description     string     `json:"Description"`
	DescriptionHook ConfigHook `json:"DescriptionHook"`

	Content     string     `json:"Content"`
	ContentHook ConfigHook `json:"ContentHook"`

	Pager      string `json:"Pager"`
	PagerLimit int    `json:"PagerLimit"`
}

type PROP_ID

type PROP_ID struct {
	Object string `json:"object"`
	Prop   string `json:"prop"`
}

type RemoteBodySanitized

type RemoteBodySanitized struct {
	Code   ErrorCode `json:"code"`
	Status string    `json:"status"`
	Date   time.Time `json:"date"`
	Body   string    `json:"data,omitempty"`
}

func MixupRemoteBodySanitized

func MixupRemoteBodySanitized(code ErrorCode, status string, date time.Time, data string) (result RemoteBodySanitized)

type RuleCache

type RuleCache struct {
	Body []byte
	Time time.Time
	Sign string
	File string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL