Documentation
¶
Index ¶
- type ArrayConfig
- type BaseField
- type BrowserConnectorConfig
- type CalculatedConfig
- type ChromiumConfig
- type CliItem
- type Config
- type Connector
- type ConnectorConfig
- type ConsoleConfig
- type DockerConfig
- type Field
- type FieldType
- type FormattedFieldConfig
- type GeneratedFieldConfig
- type HTTPTrigger
- type HostRequestLimiter
- type Item
- type Limits
- type Model
- type ModelField
- type ModelType
- type NotifierConfig
- type ObjectConfig
- type ParserType
- type PlaywrightBrowser
- type PlaywrightConfig
- type PluginConnectorConfig
- type PluginFieldConfig
- type SchedulerTrigger
- type ServerConnectorConfig
- type StaticArrayConfig
- type StaticConnectorConfig
- type StaticGeneratedFieldConfig
- type TelegramBotConfig
- type TriggerConfig
- type UUIDGeneratedFieldConfig
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type ArrayConfig ¶
type ArrayConfig struct {
RootPath string `json:"root_path" yaml:"root_path"`
ItemConfig *ObjectConfig `json:"item_config" yaml:"item_config"`
LengthLimit uint32 `json:"length_limit" yaml:"length_limit"`
StaticConfig *StaticArrayConfig `json:"static_array" yaml:"static_array"`
}
type BaseField ¶
type BaseField struct {
Type FieldType `yaml:"type" json:"type"`
Path string `yaml:"path" json:"path"`
Generated *GeneratedFieldConfig `yaml:"generated" json:"generated"`
FirstOf []*BaseField `json:"first_of" yaml:"first_of"`
}
type BrowserConnectorConfig ¶
type BrowserConnectorConfig struct {
Chromium *ChromiumConfig `json:"chromium" yaml:"chromium"`
Docker *DockerConfig `json:"docker" yaml:"docker"`
Playwright *PlaywrightConfig `json:"playwright" yaml:"playwright"`
}
type CalculatedConfig ¶ added in v0.0.19
type ChromiumConfig ¶
type ConnectorConfig ¶
type ConnectorConfig struct {
ResponseType ParserType `json:"response_type" yaml:"response_type"`
Url string `json:"url" yaml:"url"`
Attempts uint32 `json:"attempts" yaml:"attempts"`
StaticConfig *StaticConnectorConfig `json:"static_config" yaml:"static_config"`
ServerConfig *ServerConnectorConfig `json:"server_config" yaml:"server_config"`
BrowserConfig *BrowserConnectorConfig `yaml:"browser_config" json:"browser_config"`
PluginConnectorConfig *PluginConnectorConfig `json:"plugin_connector_config" yaml:"plugin_connector_config"`
}
type ConsoleConfig ¶ added in v0.0.13
type ConsoleConfig struct {
}
type DockerConfig ¶ added in v0.0.2
type DockerConfig struct {
Image string `yaml:"image" json:"image"`
EntryPoint string `json:"entry_point" yaml:"entry_point"`
Timeout uint32 `yaml:"timeout" json:"timeout"`
Wait uint32 `yaml:"wait" json:"wait"`
Flags []string `yaml:"flags" json:"flags"`
Purge bool `json:"purge" yaml:"purge"`
NoPull bool `yaml:"no_pull" json:"no_pull"`
PullTimeout uint32 `yaml:"pull_timeout" json:"pull_timeout"`
}
type Field ¶
type Field struct {
BaseField *BaseField `json:"base_field" yaml:"base_field"`
ObjectConfig *ObjectConfig `json:"object_config" yaml:"object_config"`
ArrayConfig *ArrayConfig `json:"array_config" yaml:"array_config"`
FirstOf []*Field `json:"first_of" yaml:"first_of"`
}
type FormattedFieldConfig ¶
type FormattedFieldConfig struct {
Template string `yaml:"template" json:"template"`
}
type GeneratedFieldConfig ¶
type GeneratedFieldConfig struct {
UUID *UUIDGeneratedFieldConfig `yaml:"uuid" json:"uuid"`
Static *StaticGeneratedFieldConfig `yaml:"static" json:"static"`
Formatted *FormattedFieldConfig `json:"formatted" yaml:"formatted"`
Plugin *PluginFieldConfig `yaml:"plugin" json:"plugin"`
Calculated *CalculatedConfig `yaml:"calculated" json:"calculated"`
Model *ModelField `yaml:"model" json:"model"`
}
type HTTPTrigger ¶
type HTTPTrigger struct {
}
type HostRequestLimiter ¶
type Item ¶
type Item struct {
Name string `yaml:"name" json:"name"`
// Type of parsing
ConnectorConfig *ConnectorConfig `yaml:"connector_config" json:"connector_config"`
// TriggerConfig
TriggerConfig *TriggerConfig `yaml:"trigger_config" json:"trigger_config"`
// Model of the response
Model *Model `yaml:"model" json:"model"`
// Where to report result
NotifierConfig *NotifierConfig `json:"notifier_config" yaml:"notifier_config"`
}
type Limits ¶
type Limits struct {
HostRequestLimiter HostRequestLimiter `yaml:"host_request_limiter" json:"host_request_limiter"`
ChromiumInstance uint32 `yaml:"chromium_instance" json:"chromium_instance"`
DockerContainers uint32 `yaml:"docker_containers" json:"docker_containers"`
PlaywrightInstance uint32 `yaml:"playwright_instance" json:"playwright_instance"`
}
type Model ¶
type Model struct {
ObjectConfig *ObjectConfig `yaml:"object_config" json:"object_config"`
ArrayConfig *ArrayConfig `json:"array_config" yaml:"array_config"`
BaseField *BaseField `json:"base_field" yaml:"base_field"`
}
type ModelField ¶
type ModelField struct {
// Type of parsing
ConnectorConfig *ConnectorConfig `yaml:"connector_config" json:"connector_config"`
// Model of the response
Model *Model `yaml:"model" json:"model"`
Type FieldType `yaml:"type" json:"type"`
Path string `yaml:"path" json:"path"`
}
type NotifierConfig ¶
type NotifierConfig struct {
Expression string `yaml:"expression" json:"expression"`
Force bool `json:"force" yaml:"force"`
Console *ConsoleConfig `yaml:"console" json:"console"`
TelegramBot *TelegramBotConfig `yaml:"telegram_bot" json:"telegram_bot"`
}
type ObjectConfig ¶
type ObjectConfig struct {
Field *BaseField `json:"field" yaml:"field"`
Fields map[string]*Field `json:"fields" yaml:"fields"`
ArrayConfig *ArrayConfig `json:"array_config" yaml:"array_config"`
}
type ParserType ¶
type ParserType string
const ( HTML ParserType = "HTML" Json ParserType = "json" XML ParserType = "XML" XPath ParserType = "xpath" )
type PlaywrightBrowser ¶ added in v0.0.3
type PlaywrightBrowser string
const ( Chromium PlaywrightBrowser = "Chromium" FireFox PlaywrightBrowser = "FireFox" WebKit PlaywrightBrowser = "WebKit" )
type PlaywrightConfig ¶ added in v0.0.3
type PlaywrightConfig struct {
Browser PlaywrightBrowser `json:"browser" yaml:"browser"`
Install bool `yaml:"install" json:"install"`
Timeout uint32 `yaml:"timeout" json:"timeout"`
Wait uint32 `yaml:"wait" json:"wait"`
TypeOfWait *playwright.WaitUntilState `json:"type_of_wait" yaml:"type_of_wait"`
PreRunScript string `json:"pre_run_script" yaml:"pre_run_script"`
}
type PluginConnectorConfig ¶ added in v0.0.20
type PluginConnectorConfig struct {
Name string `json:"name" yaml:"name"`
Config json.RawMessage `json:"config" yaml:"config"`
}
type PluginFieldConfig ¶ added in v0.0.16
type PluginFieldConfig struct {
Name string `json:"name" yaml:"name"`
Config json.RawMessage `json:"config" yaml:"config"`
}
type SchedulerTrigger ¶
type SchedulerTrigger struct {
// Interval for update rerun process in second
Interval int `yaml:"interval" json:"interval"`
}
type ServerConnectorConfig ¶
type StaticArrayConfig ¶ added in v0.0.5
type StaticConnectorConfig ¶ added in v0.0.10
type StaticConnectorConfig struct {
Value string `json:"value" yaml:"value"`
}
type TelegramBotConfig ¶ added in v0.0.13
type TriggerConfig ¶
type TriggerConfig struct {
SchedulerTrigger *SchedulerTrigger `yaml:"scheduler_trigger" json:"scheduler_trigger"`
HTTPTrigger *HTTPTrigger `json:"http_trigger" yaml:"http_trigger"`
}
type UUIDGeneratedFieldConfig ¶
type UUIDGeneratedFieldConfig struct {
Regexp string `yaml:"regexp" json:"regexp"`
}
Click to show internal directories.
Click to hide internal directories.