ytlocal

package
v0.0.8 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 4, 2023 License: Apache-2.0 Imports: 18 Imported by: 0

README

ytlocal

This package provides a way to launch a local version of ytsaurus cluster.

Building

First, prepare environment:

./prepare.sh

This will build the yt-builder image.

To reuse prebuilt image, skip this step:

cd _ytwork

git clone https://github.com/ytsaurus/ytsaurus.git
mkdir -p build

Then, build the package:

./build.sh

This will build the ytserver-all binary with ninja.

Resulting binary will be at /_ytwork/build/yt/yt/server/all/ytserver-all.

Documentation

Overview

Package ytlocal provides a way to run ytsaurus locally.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GenerateCellID

func GenerateCellID(cellTag int16, name string) string

GenerateCellID generates a cell ID from a cell tag and a name.

func Go

func Go(ctx context.Context, g *errgroup.Group, components ...interface {
	Run(ctx context.Context) error
	String() string
})

Go adds component to group.

Types

type AddressResolver

type AddressResolver struct {
	Retries       int    `yson:"retries,omitempty"`
	EnableIPv6    bool   `yson:"enable_ipv6"`
	EnableIPv4    bool   `yson:"enable_ipv4"`
	LocalhostFQDN string `yson:"localhost_fqdn,omitempty"`
}

AddressResolver config.

type BaseServer

type BaseServer struct {
	RPCPort           int               `yson:"rpc_port"`
	MonitoringPort    int               `yson:"monitoring_port"`
	Logging           Logging           `yson:"logging"`
	AddressResolver   AddressResolver   `yson:"address_resolver"`
	TimestampProvider Connection        `yson:"timestamp_provider"`
	ClusterConnection ClusterConnection `yson:"cluster_connection"`
	SkynetHTTPPort    int               `yson:"skynet_http_port,omitempty"`
}

BaseServer wraps common server configiguratio parts.

type Binary

type Binary struct {
	All               string
	Master            string
	Clock             string
	HTTPProxy         string
	JobProxy          string
	ControllerAgent   string
	Node              string
	TCPProxy          string
	MasterCache       string
	CellBalancer      string
	QueueAgent        string
	TimestampProvider string
	Discovery         string
	LogTailer         string
	Scheduler         string
	Tools             string
	Exec              string
	CypressProxy      string
	TabletBalancer    string
	QueryTracker      string

	// Components is a map of component name to binary path.
	Components map[Component]string
}

Binary is helper for dealing with ytsaurus binaries.

func NewBinary

func NewBinary(path string) (*Binary, error)

NewBinary initializes a set of ytsaurus binaries, using path as a directory for symlinked binaries.

type CellDirectory

type CellDirectory struct {
	SoftBackoffTime           int  `yson:"soft_backoff_time,omitempty"`
	HardBackoffTime           int  `yson:"hard_backoff_time,omitempty"`
	RetryBackoffTime          int  `yson:"retry_backoff_time,omitempty"`
	EnablePeerPolling         bool `yson:"enable_peer_polling,omitempty"`
	PeerPollingPeriod         int  `yson:"peer_polling_period,omitempty"`
	PeerPollingPeriodSplay    int  `yson:"peer_polling_period_splay,omitempty"`
	PeerPollingRequestTimeout int  `yson:"peer_polling_request_timeout,omitempty"`
	RediscoverPeriod          int  `yson:"rediscover_period,omitempty"`
	RediscoverSplay           int  `yson:"rediscover_splay,omitempty"`
}

CellDirectory config.

type ChunkClientDispatcher

type ChunkClientDispatcher struct {
	ChunkReaderPoolSize int `yson:"chunk_reader_pool_size"`
}

ChunkClientDispatcher config.

type ChunkManger

type ChunkManger struct {
	AllowMultipleErasurePartsPerNode bool `yson:"allow_multiple_erasure_parts_per_node"`
}

ChunkManger config.

type Client

type Client struct {
	AddressResolver AddressResolver `yson:"address_resolver"`
	Driver          Driver          `yson:"driver"`
}

Client config.

type ClusterConfig

type ClusterConfig struct {
	Clusters []UICluster `json:"clusters"`
}

ClusterConfig config.

type ClusterConnection

type ClusterConnection struct {
	ClusterName         string             `yson:"cluster_name"`
	DiscoveryConnection Connection         `yson:"discovery_connection"`
	PrimaryMaster       Connection         `yson:"primary_master"`
	CellDirectory       CellDirectory      `yson:"cell_directory"`
	TransactionManager  TransactionManager `yson:"transaction_manager"`
	TimestampProvider   Connection         `yson:"timestamp_provider"`
}

ClusterConnection config.

type ClusterPrimaryMaster

type ClusterPrimaryMaster struct {
	CellTag int `json:"cellTag"`
}

ClusterPrimaryMaster config.

type Component

type Component string

Component describes a component type.

const (
	ComponentHTTPProxy         Component = "http-proxy"
	ComponentMaster            Component = "master"
	ComponentNode              Component = "node"
	ComponentJobProxy          Component = "job-proxy"
	ComponentClock             Component = "clock"
	ComponentScheduler         Component = "scheduler"
	ComponentExec              Component = "exec"
	ComponentTools             Component = "tools"
	ComponentControllerAgent   Component = "controller-agent"
	ComponentLogTailer         Component = "log-tailer"
	ComponentDiscovery         Component = "discovery"
	ComponentTimestampProvider Component = "timestamp-provider"
	ComponentMasherCache       Component = "master-cache"
	ComponentCellBalancer      Component = "cell-balancer"
	ComponentQueueAgent        Component = "queue-agent"
	ComponentTabletBalancer    Component = "tablet-balancer"
	ComponentQueryTracker      Component = "query-tracker"
	ComponentCypressProxy      Component = "cypress-proxy"
	ComponentTCPProxy          Component = "tcp-proxy"
)

Component types.

type Connection

type Connection struct {
	Addresses                  []string `yson:"addresses"`
	EnableMasterCacheDiscovery bool     `yson:"enable_master_cache_discovery"`
	CellID                     string   `yson:"cell_id,omitempty"`
	SoftBackoffTime            int      `yson:"soft_backoff_time,omitempty"`
	HardBackoffTime            int      `yson:"hard_backoff_time,omitempty"`
	RetryBackoffTime           int      `yson:"retry_backoff_time,omitempty"`
	RetryAttempts              int      `yson:"retry_attempts,omitempty"`
	EnablePeerPolling          bool     `yson:"enable_peer_polling,omitempty"`
	PeerPollingPeriod          int      `yson:"peer_polling_period,omitempty"`
	PeerPollingPeriodSplay     int      `yson:"peer_polling_period_splay,omitempty"`
	PeerPollingRequestTimeout  int      `yson:"peer_polling_request_timeout,omitempty"`
	RediscoverPeriod           int      `yson:"rediscover_period,omitempty"`
	RediscoverSplay            int      `yson:"rediscover_splay,omitempty"`
	RPCTimeout                 int      `yson:"rpc_timeout,omitempty"`
	UpdatePeriod               int      `yson:"update_period,omitempty"`
}

Connection config.

type ControllerAgent

type ControllerAgent struct {
	BaseServer
	Options ControllerAgentOptions `yson:"controller_agent"`
}

ControllerAgent config.

type ControllerAgentOptions

type ControllerAgentOptions struct {
	UseColumnarStatisticsDefault bool `yson:"use_columnar_statistics_default"`
	EnableTMPFS                  bool `yson:"enable_tmpfs"`
}

ControllerAgentOptions config.

type Coordinator

type Coordinator struct {
	DefaultRoleFilter string `yson:"default_role_filter,omitempty"`
	Enable            bool   `yson:"enable"`
	Announce          bool   `yson:"announce,omitempty"`
	ShowPorts         bool   `yson:"show_ports,omitempty"`
	PublicFQDN        string `yson:"public_fqdn,omitempty"`
}

Coordinator config.

type CypressAnnotations

type CypressAnnotations struct {
	YTEnvIndex int `yson:"yt_env_index"`
}

CypressAnnotations config.

type CypressCookieManager

type CypressCookieManager struct{}

CypressCookieManager config.

type CypressManager

type CypressManager struct {
	DefaultTableReplicationFactor   int `yson:"default_table_replication_factor,omitempty"`
	DefaultFileReplicationFactor    int `yson:"default_file_replication_factor,omitempty"`
	DefaultJournalReplicationFactor int `yson:"default_journal_replication_factor,omitempty"`
	DefaultJournalReadQuorum        int `yson:"default_journal_read_quorum,omitempty"`
	DefaultJournalWriteQuorum       int `yson:"default_journal_write_quorum,omitempty"`
}

CypressManager config.

type CypressTokenAuthenticator

type CypressTokenAuthenticator struct {
	Secure bool `yson:"secure"`
}

CypressTokenAuthenticator config.

type DataNodeOptions

type DataNodeOptions struct {
	StoreLocations []StoreLocation `yson:"store_locations"`
}

DataNodeOptions config.

type Discovery

type Discovery struct {
	RPCPort         int             `yson:"rpc_port"`
	MonitoringPort  int             `yson:"monitoring_port"`
	DiscoveryServer DiscoveryServer `yson:"discovery_server"`
}

Discovery config.

type DiscoveryServer

type DiscoveryServer struct {
	ServerAddresses []string `yson:"server_addresses"`
}

DiscoveryServer config.

type Driver

type Driver struct {
	ClusterName            string        `yson:"cluster_name,omitempty"`
	MasterCache            MasterCache   `yson:"master_cache"`
	TimestampProvider      Connection    `yson:"timestamp_provider"`
	PrimaryMaster          Connection    `yson:"primary_master"`
	CellDirectory          CellDirectory `yson:"cell_directory"`
	APIVersion             int           `yson:"api_version,omitempty"`
	EnableInternalCommands bool          `yson:"enable_internal_commands"`
	TransactionManager     Connection    `yson:"transaction_manager"`
	DiscoveryConnections   []Connection  `yson:"discovery_connections,omitempty"`
}

Driver config.

type ExecAgent

type ExecAgent struct {
	SlotManager SlotManager `yson:"slot_manager"`
}

ExecAgent config.

type ExecNode

type ExecNode struct {
	BaseServer
	Flavors        []string             `yson:"flavors"`
	ResourceLimits ResourceLimits       `yson:"resource_limits"`
	TabletNode     TabletNodeConnection `yson:"tablet_node"`
	ExecAgent      ExecAgent            `yson:"exec_agent"`
}

ExecNode config.

type HTTPAuth

type HTTPAuth struct {
	RequireAuthentication     bool                      `yson:"require_authentication"`
	CypressTokenAuthenticator CypressTokenAuthenticator `yson:"cypress_token_authenticator"`
	CypressCookieManager      CypressCookieManager      `yson:"cypress_cookie_manager"`
}

HTTPAuth configures http-proxy auth.

type HTTPProxy

type HTTPProxy struct {
	BaseServer
	Port        int         `yson:"port"`
	Role        string      `yson:"role,omitempty"`
	ThreadCount int         `yson:"thread_count"`
	Driver      Driver      `yson:"driver"`
	Auth        HTTPAuth    `yson:"auth"`
	Coordinator Coordinator `yson:"coordinator,omitempty"`
}

HTTPProxy is http-proxy component config.

type HiveManager

type HiveManager struct {
	PingPeriod     int `yson:"ping_period"`
	IdlePostPeriod int `yson:"idle_post_period"`
}

HiveManager config.

type HydraManager

type HydraManager struct {
	SnapshotBackgroundThreadCount int            `yson:"snapshot_background_thread_count,omitempty"`
	LeaderSyncDelay               int            `yson:"leader_sync_delay,omitempty"`
	MinimizeCommitLatency         bool           `yson:"minimize_commit_latency"`
	LeaderLeaseCheckPeriod        int            `yson:"leader_lease_check_period,omitempty"`
	LeaderLeaseTimeout            int            `yson:"leader_lease_timeout,omitempty"`
	DisableLeaderLeaseGraceDelay  bool           `yson:"disable_leader_lease_grace_delay,omitempty"`
	InvariantsCheckProbability    float64        `yson:"invariants_check_probability"`
	MaxChangelogCountToKeep       int            `yson:"max_changelog_count_to_keep,omitempty"`
	MaxSnapshotCountToKeep        int            `yson:"max_snapshot_count_to_keep,omitempty"`
	MaxChangelogDataSize          int            `yson:"max_changelog_data_size,omitempty"`
	ResponseKeeper                ResponseKeeper `yson:"response_keeper"`
}

HydraManager config.

type IOEngine

type IOEngine struct {
	EnableSync bool `yson:"enable_sync"`
}

IOEngine is io_engine config.

type JobController

type JobController struct {
	ResourceLimits JobControllerResourceLimits `yson:"resource_limits"`
}

JobController config.

type JobControllerResourceLimits

type JobControllerResourceLimits struct {
	UserSlots int `yson:"user_slots"`
}

JobControllerResourceLimits config.

type JobEnvironment

type JobEnvironment struct {
	StartUID int64              `yson:"start_uid"`
	Type     JobEnvironmentType `yson:"type"`
}

JobEnvironment config.

type JobEnvironmentType

type JobEnvironmentType string

JobEnvironmentType is type for environment (runtime).

const (
	JobEnvironmentTypeSimple JobEnvironmentType = "simple"
	JobEnvironmentTypePorto  JobEnvironmentType = "porto"
)

Possible types.

type LogFormat

type LogFormat string

LogFormat is a string describing possible log formats.

const (
	LogFormatPlainText LogFormat = "plain_text"
	LogFormatJSON      LogFormat = "json"
	LogFormatYSON      LogFormat = "yson"
)

Possible log formats.

type LogLevel

type LogLevel string

LogLevel string describes possible logging levels.

const (
	LogLevelTrace   LogLevel = "trace"
	LogLevelDebug   LogLevel = "debug"
	LogLevelInfo    LogLevel = "info"
	LogLevenWarning LogLevel = "warning"
	LogLevelError   LogLevel = "error"
)

type LogWriterType

type LogWriterType string

LogWriterType string describes types of possible log writers.

const (
	LogWriterTypeFile   LogWriterType = "file"
	LogWriterTypeStderr LogWriterType = "stderr"
)

Possible log writer types.

type Logging

type Logging struct {
	AbortOnAlert           bool                     `yson:"abort_on_alert,omitempty"`
	CompressionThreadCount int                      `yson:"compression_thread_count,omitempty"`
	Writers                map[string]LoggingWriter `yson:"writers,omitempty"`
	Rules                  []LoggingRule            `yson:"rules,omitempty"`
}

Logging config.

type LoggingRule

type LoggingRule struct {
	ExcludeCategories []string `yson:"exclude_categories,omitempty"`
	IncludeCategories []string `yson:"include_categories,omitempty"`
	MinLevel          LogLevel `yson:"min_level,omitempty"`
	Writers           []string `yson:"writers,omitempty"`
	Family            string   `yson:"family,omitempty"`
}

LoggingRule configures logging.

type LoggingWriter

type LoggingWriter struct {
	WriterType LogWriterType `yson:"type,omitempty"`
	Format     LogFormat     `yson:"format,omitempty"`
	FileName   string        `yson:"file_name,omitempty"`
}

LoggingWriter configures logging writer.

type Master

type Master struct {
	BaseServer
	PrimaryMaster                    Connection            `yson:"primary_master"`
	UseNewHydra                      bool                  `yson:"use_new_hydra"`
	EnableProvisionLock              bool                  `yson:"enable_provision_lock"`
	Changelogs                       MasterChangelogs      `yson:"changelogs"`
	ObjectService                    ObjectService         `yson:"object_service"`
	Snapshots                        MasterSnapshots       `yson:"snapshots"`
	HydraManager                     HydraManager          `yson:"hydra_manager"`
	YPServiceDiscovery               YPServiceDiscovery    `yson:"yp_service_discovery"`
	RPCDispatcher                    RPCDispatcher         `yson:"rpc_dispatcher"`
	ChunkClientDispatcher            ChunkClientDispatcher `yson:"chunk_client_dispatcher"`
	TCPDispatcher                    TCPDispatcher         `yson:"tcp_dispatcher"`
	SolomonExporter                  SolomonExporter       `yson:"solomon_exporter"`
	CypressAnnotations               CypressAnnotations    `yson:"cypress_annotations"`
	EnableRefCountedTrackerProfiling bool                  `yson:"enable_ref_counted_tracker_profiling"`
	EnableResourceTracker            bool                  `yson:"enable_resource_tracker"`
	EnableTimestampManager           bool                  `yson:"enable_timestamp_manager"`
	TimestampManager                 TimestampManager      `yson:"timestamp_manager"`
	HiveManager                      HiveManager           `yson:"hive_manager"`
	CypressManager                   CypressManager        `yson:"cypress_manager"`
	SecondaryMasters                 []Connection          `yson:"secondary_masters"`
	ChunkManger                      ChunkManger           `yson:"chunk_manager"`
}

Master config.

type MasterCache

type MasterCache struct {
	EnableMasterCacheDiscovery bool     `yson:"enable_master_cache_discovery"`
	Addresses                  []string `yson:"addresses"`
	CellID                     string   `yson:"cell_id,omitempty"`
}

MasterCache config.

type MasterCacheService

type MasterCacheService struct {
	Capacity int64 `yson:"capaticy"`
}

MasterCacheService config.

type MasterChangelogs

type MasterChangelogs struct {
	Path        string   `yson:"path"`
	FlushPeriod int      `yson:"flush_period,omitempty"`
	EnableSync  bool     `yson:"enable_sync"`
	IOEngine    IOEngine `yson:"io_engine"`
}

MasterChangelogs config.

type MasterSnapshots

type MasterSnapshots struct {
	Path string `yson:"path"`
}

MasterSnapshots config.

type Node

type Node struct {
	BaseServer
	Flavors        []string        `yson:"flavors,omitempty"`
	ResourceLimits ResourceLimits  `yson:"resource_limits"`
	Options        DataNodeOptions `yson:"data_node"`
	Addresses      [][]string      `yson:"addresses,omitempty"`
}

Node config.

type ObjectService

type ObjectService struct {
	EnableLocalReadExecutor bool `yson:"enable_local_read_executor"`
	EnableLocalReadBusyWait bool `yson:"enable_local_read_busy_wait"`
}

ObjectService is master object service config.

type Options

type Options struct {
	Binary *Binary
	Dir    string
}

Options describes options for creating a new component server.

type PortAllocator

type PortAllocator struct {
	Host string
	Net  string
	// contains filtered or unexported fields
}

PortAllocator is a helper to allocate random ports.

func (*PortAllocator) Allocate

func (p *PortAllocator) Allocate() (int, error)

Allocate a new random port.

Tries to allocate a new port 10 times before returning an error. Safe to call concurrently.

type QueryTracker

type QueryTracker struct {
	BaseServer
	User                       string `yson:"user"`
	CreateStateTablesOnStartup bool   `yson:"create_state_tables_on_startup"`
}

QueryTracker config.

type RPCDispatcher

type RPCDispatcher struct {
	CompressionPoolSize int `yson:"compression_pool_size"`
	HeavyPoolSize       int `yson:"heavy_pool_size"`
}

RPCDispatcher is rpc_dispatcher config.

type RPCProxy

type RPCProxy struct {
	BaseServer
	Role                      string                    `yson:"role"`
	CypressTokenAuthenticator CypressTokenAuthenticator `yson:"cypress_token_authenticator"`
}

RPCProxy is rpc-proxy config.

type ResourceLimits

type ResourceLimits struct {
	TotalCPU         float64 `yson:"total_cpu,omitempty"`
	TotalMemory      int64   `yson:"total_memory,omitempty"`
	NodeDedicatedCPU float64 `yson:"node_dedicated_cpu,omitempty"`
	Memory           int64   `yson:"memory,omitempty"`
}

ResourceLimits config.

type ResponseKeeper

type ResponseKeeper struct {
	EnableWarmup   bool `yson:"enable_warmup"`
	ExpirationTime int  `yson:"expiration_time"`
	WarmupTime     int  `yson:"warmup_time"`
}

ResponseKeeper is hydra manager response keeper config.

type Scheduler

type Scheduler struct {
	BaseServer
}

Scheduler config.

type Server

type Server[T any] struct {
	Type   Component
	Config T
	Binary string
	Dir    string
}

Server describes a component server.

func NewComponent

func NewComponent[T any](opt Options, cfg T) *Server[T]

NewComponent creates a new component server.

func (*Server[T]) Run

func (s *Server[T]) Run(ctx context.Context) error

Run runs a component server.

func (Server[T]) String

func (s Server[T]) String() string

type SlotLocation

type SlotLocation struct {
	Path               string `yson:"path"`
	DiskQuota          int64  `yson:"disk_quota"`
	DiskUsageWatermark int64  `yson:"disk_usage_watermark"`
}

SlotLocation config.

type SlotManager

type SlotManager struct {
	JobEnvironment JobEnvironment `yson:"job_environment"`
	Locations      []SlotLocation `yson:"locations"`
}

SlotManager config.

type SolomonExporter

type SolomonExporter struct {
	GridStep int `yson:"grid_step"`
}

SolomonExporter config.

type StoreLocation

type StoreLocation struct {
	Quota                  int64  `yson:"quota,omitempty"`
	MediumName             string `yson:"medium_name,omitempty"`
	LowWatermark           int64  `yson:"low_watermark,omitempty"`
	DisableWritesWatermark int64  `yson:"disable_writes_watermark,omitempty"`
	Path                   string `yson:"path,omitempty"`
	HighWatermark          int64  `yson:"high_watermark,omitempty"`
}

StoreLocation config.

type TCPDispatcher

type TCPDispatcher struct {
	ThreadPoolSize int `yson:"thread_pool_size"`
}

TCPDispatcher is tcp_dispatcher config.

type TabletNode

type TabletNode struct {
	BaseServer
	Flavors        []string       `yson:"flavors"`
	ResourceLimits ResourceLimits `yson:"resource_limits"`
}

TabletNode is tabled-node config.

type TabletNodeConnection

type TabletNodeConnection struct {
	VersionedChunkMetaCache VersionedChunkMetaCache `yson:"versioned_chunk_meta_cache"`
}

TabletNodeConnection config.

type TimestampManager

type TimestampManager struct {
	CommitAdvance      int `yson:"commit_advance"`
	RequestBackoffTime int `yson:"request_backoff_time"`
	CalibrationPeriod  int `yson:"calibration_period"`
}

TimestampManager config.

type TransactionManager

type TransactionManager struct {
	DefaultPingPeriod int `yson:"default_ping_period"`
}

TransactionManager config.

type UICluster

type UICluster struct {
	ID             string               `json:"id"`
	Name           string               `json:"name,omitempty"`
	Proxy          string               `json:"proxy,omitempty"`
	Secure         bool                 `json:"secure"`
	Theme          string               `json:"theme,omitempty"`
	Authentication string               `json:"authentication"`
	Group          string               `json:"group,omitempty"`
	Environment    string               `json:"environment,omitempty"`
	Description    string               `json:"description,omitempty"`
	PrimaryMaster  ClusterPrimaryMaster `json:"primaryMaster"`
}

UICluster is a cluster config for UI.

type VersionedChunkMetaCache

type VersionedChunkMetaCache struct {
	Capacity int `yson:"capacity"`
}

VersionedChunkMetaCache config.

type YPServiceDiscovery

type YPServiceDiscovery struct {
	Enable bool `yson:"enable"`
}

YPServiceDiscovery is yp_service_discovery config.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL