Documentation
¶
Overview ¶
Package ytlocal provides a way to run ytsaurus locally.
Index ¶
- func GenerateCellID(cellTag int16, name string) string
- func Go(ctx context.Context, g *errgroup.Group, components ...)
- type AddressResolver
- type BaseServer
- type Binary
- type CellDirectory
- type ChunkClientDispatcher
- type ChunkManger
- type Client
- type ClusterConfig
- type ClusterConnection
- type ClusterPrimaryMaster
- type Component
- type Connection
- type ControllerAgent
- type ControllerAgentOptions
- type Coordinator
- type CypressAnnotations
- type CypressCookieManager
- type CypressManager
- type CypressTokenAuthenticator
- type DataNodeOptions
- type Discovery
- type DiscoveryServer
- type Driver
- type ExecAgent
- type ExecNode
- type HTTPAuth
- type HTTPProxy
- type HiveManager
- type HydraManager
- type IOEngine
- type JobController
- type JobControllerResourceLimits
- type JobEnvironment
- type JobEnvironmentType
- type LogFormat
- type LogLevel
- type LogWriterType
- type Logging
- type LoggingRule
- type LoggingWriter
- type Master
- type MasterCache
- type MasterCacheService
- type MasterChangelogs
- type MasterSnapshots
- type Node
- type ObjectService
- type Options
- type PortAllocator
- type QueryTracker
- type RPCDispatcher
- type RPCProxy
- type ResourceLimits
- type ResponseKeeper
- type Scheduler
- type Server
- type SlotLocation
- type SlotManager
- type SolomonExporter
- type StoreLocation
- type TCPDispatcher
- type TabletNode
- type TabletNodeConnection
- type TimestampManager
- type TransactionManager
- type UICluster
- type VersionedChunkMetaCache
- type YPServiceDiscovery
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GenerateCellID ¶
GenerateCellID generates a cell ID from a cell tag and a name.
Types ¶
type AddressResolver ¶
type AddressResolver struct {
Retries int `yson:"retries,omitempty"`
EnableIPv6 bool `yson:"enable_ipv6"`
EnableIPv4 bool `yson:"enable_ipv4"`
LocalhostFQDN string `yson:"localhost_fqdn,omitempty"`
}
AddressResolver config.
type BaseServer ¶
type BaseServer struct {
RPCPort int `yson:"rpc_port"`
MonitoringPort int `yson:"monitoring_port"`
Logging Logging `yson:"logging"`
AddressResolver AddressResolver `yson:"address_resolver"`
TimestampProvider Connection `yson:"timestamp_provider"`
ClusterConnection ClusterConnection `yson:"cluster_connection"`
SkynetHTTPPort int `yson:"skynet_http_port,omitempty"`
}
BaseServer wraps common server configiguratio parts.
type Binary ¶
type Binary struct {
All string
Master string
Clock string
HTTPProxy string
JobProxy string
ControllerAgent string
Node string
TCPProxy string
MasterCache string
CellBalancer string
QueueAgent string
TimestampProvider string
Discovery string
LogTailer string
Scheduler string
Tools string
Exec string
CypressProxy string
TabletBalancer string
QueryTracker string
// Components is a map of component name to binary path.
Components map[Component]string
}
Binary is helper for dealing with ytsaurus binaries.
type CellDirectory ¶
type CellDirectory struct {
SoftBackoffTime int `yson:"soft_backoff_time,omitempty"`
HardBackoffTime int `yson:"hard_backoff_time,omitempty"`
RetryBackoffTime int `yson:"retry_backoff_time,omitempty"`
EnablePeerPolling bool `yson:"enable_peer_polling,omitempty"`
PeerPollingPeriod int `yson:"peer_polling_period,omitempty"`
PeerPollingPeriodSplay int `yson:"peer_polling_period_splay,omitempty"`
PeerPollingRequestTimeout int `yson:"peer_polling_request_timeout,omitempty"`
RediscoverPeriod int `yson:"rediscover_period,omitempty"`
RediscoverSplay int `yson:"rediscover_splay,omitempty"`
}
CellDirectory config.
type ChunkClientDispatcher ¶
type ChunkClientDispatcher struct {
ChunkReaderPoolSize int `yson:"chunk_reader_pool_size"`
}
ChunkClientDispatcher config.
type ChunkManger ¶
type ChunkManger struct {
AllowMultipleErasurePartsPerNode bool `yson:"allow_multiple_erasure_parts_per_node"`
}
ChunkManger config.
type Client ¶
type Client struct {
AddressResolver AddressResolver `yson:"address_resolver"`
Driver Driver `yson:"driver"`
}
Client config.
type ClusterConfig ¶
type ClusterConfig struct {
Clusters []UICluster `json:"clusters"`
}
ClusterConfig config.
type ClusterConnection ¶
type ClusterConnection struct {
ClusterName string `yson:"cluster_name"`
DiscoveryConnection Connection `yson:"discovery_connection"`
PrimaryMaster Connection `yson:"primary_master"`
CellDirectory CellDirectory `yson:"cell_directory"`
TransactionManager TransactionManager `yson:"transaction_manager"`
TimestampProvider Connection `yson:"timestamp_provider"`
}
ClusterConnection config.
type ClusterPrimaryMaster ¶
type ClusterPrimaryMaster struct {
CellTag int `json:"cellTag"`
}
ClusterPrimaryMaster config.
type Component ¶
type Component string
Component describes a component type.
const ( ComponentHTTPProxy Component = "http-proxy" ComponentMaster Component = "master" ComponentNode Component = "node" ComponentJobProxy Component = "job-proxy" ComponentClock Component = "clock" ComponentScheduler Component = "scheduler" ComponentExec Component = "exec" ComponentTools Component = "tools" ComponentControllerAgent Component = "controller-agent" ComponentLogTailer Component = "log-tailer" ComponentDiscovery Component = "discovery" ComponentTimestampProvider Component = "timestamp-provider" ComponentMasherCache Component = "master-cache" ComponentCellBalancer Component = "cell-balancer" ComponentQueueAgent Component = "queue-agent" ComponentTabletBalancer Component = "tablet-balancer" ComponentQueryTracker Component = "query-tracker" ComponentCypressProxy Component = "cypress-proxy" ComponentTCPProxy Component = "tcp-proxy" )
Component types.
type Connection ¶
type Connection struct {
Addresses []string `yson:"addresses"`
EnableMasterCacheDiscovery bool `yson:"enable_master_cache_discovery"`
CellID string `yson:"cell_id,omitempty"`
SoftBackoffTime int `yson:"soft_backoff_time,omitempty"`
HardBackoffTime int `yson:"hard_backoff_time,omitempty"`
RetryBackoffTime int `yson:"retry_backoff_time,omitempty"`
RetryAttempts int `yson:"retry_attempts,omitempty"`
EnablePeerPolling bool `yson:"enable_peer_polling,omitempty"`
PeerPollingPeriod int `yson:"peer_polling_period,omitempty"`
PeerPollingPeriodSplay int `yson:"peer_polling_period_splay,omitempty"`
PeerPollingRequestTimeout int `yson:"peer_polling_request_timeout,omitempty"`
RediscoverPeriod int `yson:"rediscover_period,omitempty"`
RediscoverSplay int `yson:"rediscover_splay,omitempty"`
RPCTimeout int `yson:"rpc_timeout,omitempty"`
UpdatePeriod int `yson:"update_period,omitempty"`
}
Connection config.
type ControllerAgent ¶
type ControllerAgent struct {
BaseServer
Options ControllerAgentOptions `yson:"controller_agent"`
}
ControllerAgent config.
type ControllerAgentOptions ¶
type ControllerAgentOptions struct {
UseColumnarStatisticsDefault bool `yson:"use_columnar_statistics_default"`
EnableTMPFS bool `yson:"enable_tmpfs"`
}
ControllerAgentOptions config.
type Coordinator ¶
type Coordinator struct {
DefaultRoleFilter string `yson:"default_role_filter,omitempty"`
Enable bool `yson:"enable"`
Announce bool `yson:"announce,omitempty"`
ShowPorts bool `yson:"show_ports,omitempty"`
PublicFQDN string `yson:"public_fqdn,omitempty"`
}
Coordinator config.
type CypressAnnotations ¶
type CypressAnnotations struct {
YTEnvIndex int `yson:"yt_env_index"`
}
CypressAnnotations config.
type CypressManager ¶
type CypressManager struct {
DefaultTableReplicationFactor int `yson:"default_table_replication_factor,omitempty"`
DefaultFileReplicationFactor int `yson:"default_file_replication_factor,omitempty"`
DefaultJournalReplicationFactor int `yson:"default_journal_replication_factor,omitempty"`
DefaultJournalReadQuorum int `yson:"default_journal_read_quorum,omitempty"`
DefaultJournalWriteQuorum int `yson:"default_journal_write_quorum,omitempty"`
}
CypressManager config.
type CypressTokenAuthenticator ¶
type CypressTokenAuthenticator struct {
Secure bool `yson:"secure"`
}
CypressTokenAuthenticator config.
type DataNodeOptions ¶
type DataNodeOptions struct {
StoreLocations []StoreLocation `yson:"store_locations"`
}
DataNodeOptions config.
type Discovery ¶
type Discovery struct {
RPCPort int `yson:"rpc_port"`
MonitoringPort int `yson:"monitoring_port"`
DiscoveryServer DiscoveryServer `yson:"discovery_server"`
}
Discovery config.
type DiscoveryServer ¶
type DiscoveryServer struct {
ServerAddresses []string `yson:"server_addresses"`
}
DiscoveryServer config.
type Driver ¶
type Driver struct {
ClusterName string `yson:"cluster_name,omitempty"`
MasterCache MasterCache `yson:"master_cache"`
TimestampProvider Connection `yson:"timestamp_provider"`
PrimaryMaster Connection `yson:"primary_master"`
CellDirectory CellDirectory `yson:"cell_directory"`
APIVersion int `yson:"api_version,omitempty"`
EnableInternalCommands bool `yson:"enable_internal_commands"`
TransactionManager Connection `yson:"transaction_manager"`
DiscoveryConnections []Connection `yson:"discovery_connections,omitempty"`
}
Driver config.
type ExecAgent ¶
type ExecAgent struct {
SlotManager SlotManager `yson:"slot_manager"`
}
ExecAgent config.
type ExecNode ¶
type ExecNode struct {
BaseServer
Flavors []string `yson:"flavors"`
ResourceLimits ResourceLimits `yson:"resource_limits"`
TabletNode TabletNodeConnection `yson:"tablet_node"`
ExecAgent ExecAgent `yson:"exec_agent"`
}
ExecNode config.
type HTTPAuth ¶
type HTTPAuth struct {
RequireAuthentication bool `yson:"require_authentication"`
CypressTokenAuthenticator CypressTokenAuthenticator `yson:"cypress_token_authenticator"`
CypressCookieManager CypressCookieManager `yson:"cypress_cookie_manager"`
}
HTTPAuth configures http-proxy auth.
type HTTPProxy ¶
type HTTPProxy struct {
BaseServer
Port int `yson:"port"`
Role string `yson:"role,omitempty"`
ThreadCount int `yson:"thread_count"`
Driver Driver `yson:"driver"`
Auth HTTPAuth `yson:"auth"`
Coordinator Coordinator `yson:"coordinator,omitempty"`
}
HTTPProxy is http-proxy component config.
type HiveManager ¶
type HiveManager struct {
PingPeriod int `yson:"ping_period"`
IdlePostPeriod int `yson:"idle_post_period"`
}
HiveManager config.
type HydraManager ¶
type HydraManager struct {
SnapshotBackgroundThreadCount int `yson:"snapshot_background_thread_count,omitempty"`
LeaderSyncDelay int `yson:"leader_sync_delay,omitempty"`
MinimizeCommitLatency bool `yson:"minimize_commit_latency"`
LeaderLeaseCheckPeriod int `yson:"leader_lease_check_period,omitempty"`
LeaderLeaseTimeout int `yson:"leader_lease_timeout,omitempty"`
DisableLeaderLeaseGraceDelay bool `yson:"disable_leader_lease_grace_delay,omitempty"`
InvariantsCheckProbability float64 `yson:"invariants_check_probability"`
MaxChangelogCountToKeep int `yson:"max_changelog_count_to_keep,omitempty"`
MaxSnapshotCountToKeep int `yson:"max_snapshot_count_to_keep,omitempty"`
MaxChangelogDataSize int `yson:"max_changelog_data_size,omitempty"`
ResponseKeeper ResponseKeeper `yson:"response_keeper"`
}
HydraManager config.
type IOEngine ¶
type IOEngine struct {
EnableSync bool `yson:"enable_sync"`
}
IOEngine is io_engine config.
type JobController ¶
type JobController struct {
ResourceLimits JobControllerResourceLimits `yson:"resource_limits"`
}
JobController config.
type JobControllerResourceLimits ¶
type JobControllerResourceLimits struct {
UserSlots int `yson:"user_slots"`
}
JobControllerResourceLimits config.
type JobEnvironment ¶
type JobEnvironment struct {
StartUID int64 `yson:"start_uid"`
Type JobEnvironmentType `yson:"type"`
}
JobEnvironment config.
type JobEnvironmentType ¶
type JobEnvironmentType string
JobEnvironmentType is type for environment (runtime).
const ( JobEnvironmentTypeSimple JobEnvironmentType = "simple" JobEnvironmentTypePorto JobEnvironmentType = "porto" )
Possible types.
type LogWriterType ¶
type LogWriterType string
LogWriterType string describes types of possible log writers.
const ( LogWriterTypeFile LogWriterType = "file" LogWriterTypeStderr LogWriterType = "stderr" )
Possible log writer types.
type Logging ¶
type Logging struct {
AbortOnAlert bool `yson:"abort_on_alert,omitempty"`
CompressionThreadCount int `yson:"compression_thread_count,omitempty"`
Writers map[string]LoggingWriter `yson:"writers,omitempty"`
Rules []LoggingRule `yson:"rules,omitempty"`
}
Logging config.
type LoggingRule ¶
type LoggingRule struct {
ExcludeCategories []string `yson:"exclude_categories,omitempty"`
IncludeCategories []string `yson:"include_categories,omitempty"`
MinLevel LogLevel `yson:"min_level,omitempty"`
Writers []string `yson:"writers,omitempty"`
Family string `yson:"family,omitempty"`
}
LoggingRule configures logging.
type LoggingWriter ¶
type LoggingWriter struct {
WriterType LogWriterType `yson:"type,omitempty"`
Format LogFormat `yson:"format,omitempty"`
FileName string `yson:"file_name,omitempty"`
}
LoggingWriter configures logging writer.
type Master ¶
type Master struct {
BaseServer
PrimaryMaster Connection `yson:"primary_master"`
UseNewHydra bool `yson:"use_new_hydra"`
EnableProvisionLock bool `yson:"enable_provision_lock"`
Changelogs MasterChangelogs `yson:"changelogs"`
ObjectService ObjectService `yson:"object_service"`
Snapshots MasterSnapshots `yson:"snapshots"`
HydraManager HydraManager `yson:"hydra_manager"`
YPServiceDiscovery YPServiceDiscovery `yson:"yp_service_discovery"`
RPCDispatcher RPCDispatcher `yson:"rpc_dispatcher"`
ChunkClientDispatcher ChunkClientDispatcher `yson:"chunk_client_dispatcher"`
TCPDispatcher TCPDispatcher `yson:"tcp_dispatcher"`
SolomonExporter SolomonExporter `yson:"solomon_exporter"`
CypressAnnotations CypressAnnotations `yson:"cypress_annotations"`
EnableRefCountedTrackerProfiling bool `yson:"enable_ref_counted_tracker_profiling"`
EnableResourceTracker bool `yson:"enable_resource_tracker"`
EnableTimestampManager bool `yson:"enable_timestamp_manager"`
TimestampManager TimestampManager `yson:"timestamp_manager"`
HiveManager HiveManager `yson:"hive_manager"`
CypressManager CypressManager `yson:"cypress_manager"`
SecondaryMasters []Connection `yson:"secondary_masters"`
ChunkManger ChunkManger `yson:"chunk_manager"`
}
Master config.
type MasterCache ¶
type MasterCache struct {
EnableMasterCacheDiscovery bool `yson:"enable_master_cache_discovery"`
Addresses []string `yson:"addresses"`
CellID string `yson:"cell_id,omitempty"`
}
MasterCache config.
type MasterCacheService ¶
type MasterCacheService struct {
Capacity int64 `yson:"capaticy"`
}
MasterCacheService config.
type MasterChangelogs ¶
type MasterChangelogs struct {
Path string `yson:"path"`
FlushPeriod int `yson:"flush_period,omitempty"`
EnableSync bool `yson:"enable_sync"`
IOEngine IOEngine `yson:"io_engine"`
}
MasterChangelogs config.
type MasterSnapshots ¶
type MasterSnapshots struct {
Path string `yson:"path"`
}
MasterSnapshots config.
type Node ¶
type Node struct {
BaseServer
Flavors []string `yson:"flavors,omitempty"`
ResourceLimits ResourceLimits `yson:"resource_limits"`
Options DataNodeOptions `yson:"data_node"`
Addresses [][]string `yson:"addresses,omitempty"`
}
Node config.
type ObjectService ¶
type ObjectService struct {
EnableLocalReadExecutor bool `yson:"enable_local_read_executor"`
EnableLocalReadBusyWait bool `yson:"enable_local_read_busy_wait"`
}
ObjectService is master object service config.
type PortAllocator ¶
PortAllocator is a helper to allocate random ports.
func (*PortAllocator) Allocate ¶
func (p *PortAllocator) Allocate() (int, error)
Allocate a new random port.
Tries to allocate a new port 10 times before returning an error. Safe to call concurrently.
type QueryTracker ¶
type QueryTracker struct {
BaseServer
User string `yson:"user"`
CreateStateTablesOnStartup bool `yson:"create_state_tables_on_startup"`
}
QueryTracker config.
type RPCDispatcher ¶
type RPCDispatcher struct {
CompressionPoolSize int `yson:"compression_pool_size"`
HeavyPoolSize int `yson:"heavy_pool_size"`
}
RPCDispatcher is rpc_dispatcher config.
type RPCProxy ¶
type RPCProxy struct {
BaseServer
Role string `yson:"role"`
CypressTokenAuthenticator CypressTokenAuthenticator `yson:"cypress_token_authenticator"`
}
RPCProxy is rpc-proxy config.
type ResourceLimits ¶
type ResourceLimits struct {
TotalCPU float64 `yson:"total_cpu,omitempty"`
TotalMemory int64 `yson:"total_memory,omitempty"`
NodeDedicatedCPU float64 `yson:"node_dedicated_cpu,omitempty"`
Memory int64 `yson:"memory,omitempty"`
}
ResourceLimits config.
type ResponseKeeper ¶
type ResponseKeeper struct {
EnableWarmup bool `yson:"enable_warmup"`
ExpirationTime int `yson:"expiration_time"`
WarmupTime int `yson:"warmup_time"`
}
ResponseKeeper is hydra manager response keeper config.
type Server ¶
Server describes a component server.
func NewComponent ¶
NewComponent creates a new component server.
type SlotLocation ¶
type SlotLocation struct {
Path string `yson:"path"`
DiskQuota int64 `yson:"disk_quota"`
DiskUsageWatermark int64 `yson:"disk_usage_watermark"`
}
SlotLocation config.
type SlotManager ¶
type SlotManager struct {
JobEnvironment JobEnvironment `yson:"job_environment"`
Locations []SlotLocation `yson:"locations"`
}
SlotManager config.
type SolomonExporter ¶
type SolomonExporter struct {
GridStep int `yson:"grid_step"`
}
SolomonExporter config.
type StoreLocation ¶
type StoreLocation struct {
Quota int64 `yson:"quota,omitempty"`
MediumName string `yson:"medium_name,omitempty"`
LowWatermark int64 `yson:"low_watermark,omitempty"`
DisableWritesWatermark int64 `yson:"disable_writes_watermark,omitempty"`
Path string `yson:"path,omitempty"`
HighWatermark int64 `yson:"high_watermark,omitempty"`
}
StoreLocation config.
type TCPDispatcher ¶
type TCPDispatcher struct {
ThreadPoolSize int `yson:"thread_pool_size"`
}
TCPDispatcher is tcp_dispatcher config.
type TabletNode ¶
type TabletNode struct {
BaseServer
Flavors []string `yson:"flavors"`
ResourceLimits ResourceLimits `yson:"resource_limits"`
}
TabletNode is tabled-node config.
type TabletNodeConnection ¶
type TabletNodeConnection struct {
VersionedChunkMetaCache VersionedChunkMetaCache `yson:"versioned_chunk_meta_cache"`
}
TabletNodeConnection config.
type TimestampManager ¶
type TimestampManager struct {
CommitAdvance int `yson:"commit_advance"`
RequestBackoffTime int `yson:"request_backoff_time"`
CalibrationPeriod int `yson:"calibration_period"`
}
TimestampManager config.
type TransactionManager ¶
type TransactionManager struct {
DefaultPingPeriod int `yson:"default_ping_period"`
}
TransactionManager config.
type UICluster ¶
type UICluster struct {
ID string `json:"id"`
Name string `json:"name,omitempty"`
Proxy string `json:"proxy,omitempty"`
Secure bool `json:"secure"`
Theme string `json:"theme,omitempty"`
Authentication string `json:"authentication"`
Group string `json:"group,omitempty"`
Environment string `json:"environment,omitempty"`
Description string `json:"description,omitempty"`
PrimaryMaster ClusterPrimaryMaster `json:"primaryMaster"`
}
UICluster is a cluster config for UI.
type VersionedChunkMetaCache ¶
type VersionedChunkMetaCache struct {
Capacity int `yson:"capacity"`
}
VersionedChunkMetaCache config.
type YPServiceDiscovery ¶
type YPServiceDiscovery struct {
Enable bool `yson:"enable"`
}
YPServiceDiscovery is yp_service_discovery config.