ossreader

package
v1.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 29, 2025 License: MIT Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// OSS连接配置
	KeyEndpoint  = "endpoint"
	KeyAccessId  = "accessId"
	KeyAccessKey = "accessKey"
	KeyBucket    = "bucket"
	KeyObject    = "object"
	KeyCname     = "cname"

	// 文件处理配置
	KeyColumn         = "column"
	KeyFieldDelimiter = "fieldDelimiter"
	KeyCompress       = "compress"
	KeyEncoding       = "encoding"
	KeySkipHeader     = "skipHeader"
	KeyNullFormat     = "nullFormat"
	KeyDateFormat     = "dateFormat"
	KeyFileFormat     = "fileFormat"

	// 高级配置
	KeySuccessOnNoObject = "successOnNoObject"
	KeyProxyHost         = "proxyHost"
	KeyProxyPort         = "proxyPort"
	KeyProxyUsername     = "proxyUsername"
	KeyProxyPassword     = "proxyPassword"
	KeyProxyDomain       = "proxyDomain"
	KeyProxyWorkstation  = "proxyWorkstation"

	// 分片配置
	KeyBalanceThreshold = "balanceThreshold"

	// 默认值
	DefaultFileFormat        = "text"
	DefaultFieldDelimiter    = ","
	DefaultEncoding          = "UTF-8"
	DefaultNullFormat        = "\\N"
	DefaultCompress          = ""
	DefaultSkipHeader        = false
	DefaultSuccessOnNoObject = false
	DefaultBalanceThreshold  = 0.3

	// 分片阈值常量
	SingleFileSplitThresholdInSize = 1024 * 1024 * 1024 // 1GB
)

配置项常量,与Java版本保持一致

Variables

This section is empty.

Functions

This section is empty.

Types

type ObjectSizePair

type ObjectSizePair struct {
	Key  string
	Size int64
}

ObjectSizePair 对象及其大小信息

type OssReaderJob

type OssReaderJob struct {
	// contains filtered or unexported fields
}

OssReaderJob OSS读取作业,完全复制Java版本的功能

func NewOssReaderJob

func NewOssReaderJob() *OssReaderJob

func (*OssReaderJob) Destroy

func (job *OssReaderJob) Destroy() error

func (*OssReaderJob) Init

func (job *OssReaderJob) Init(config config.Configuration) error

func (*OssReaderJob) Post

func (job *OssReaderJob) Post() error

func (*OssReaderJob) Prepare

func (job *OssReaderJob) Prepare() error

func (*OssReaderJob) Split

func (job *OssReaderJob) Split(adviceNumber int) ([]config.Configuration, error)

type OssReaderJobFactory

type OssReaderJobFactory struct{}

OssReaderJobFactory 实现ReaderJobFactory接口

func (*OssReaderJobFactory) CreateReaderJob

func (f *OssReaderJobFactory) CreateReaderJob() plugin.ReaderJob

type OssReaderTask

type OssReaderTask struct {
	// contains filtered or unexported fields
}

OssReaderTask OSS读取任务

func NewOssReaderTask

func NewOssReaderTask() *OssReaderTask

func (*OssReaderTask) Destroy

func (task *OssReaderTask) Destroy() error

func (*OssReaderTask) Init

func (task *OssReaderTask) Init(config config.Configuration) error

func (*OssReaderTask) Post

func (task *OssReaderTask) Post() error

func (*OssReaderTask) Prepare

func (task *OssReaderTask) Prepare() error

func (*OssReaderTask) StartRead

func (task *OssReaderTask) StartRead(recordSender plugin.RecordSender) error

type OssReaderTaskFactory

type OssReaderTaskFactory struct{}

OssReaderTaskFactory 实现ReaderTaskFactory接口

func (*OssReaderTaskFactory) CreateReaderTask

func (f *OssReaderTaskFactory) CreateReaderTask() plugin.ReaderTask

type SliceConfig

type SliceConfig struct {
	FilePath string `json:"filePath"`
	Start    int64  `json:"start"`
	End      int64  `json:"end"`
}

SliceConfig 切片配置

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL