helper

type ApiAsyncCallback func(*http.Response, error) error

type ApiAsyncClient struct {
	*ApiClient
	// contains filtered or unexported fields
}

type ApiClient struct {
	// contains filtered or unexported fields
}

type ApiClientAfterResponse func(res *http.Response) error

type ApiClientBeforeRequest func(req *http.Request) error

type ApiCollector struct {
	*RawDataSubTask
	// contains filtered or unexported fields
}

type ApiCollectorArgs struct {
	RawDataSubTaskArgs
	/*
		url may use arbitrary variables from different source in any order, we need GoTemplate to allow more
		flexible for all kinds of possibility.
		Pager contains information for a particular page, calculated by ApiCollector, and will be passed into
		GoTemplate to generate a url for that page.
		We want to do page-fetching in ApiCollector, because the logic are highly similar, by doing so, we can
		avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can
		do it in one place
	*/
	UrlTemplate string `comment:"GoTemplate for API url"`
	// (Optional) Return query string for request, or you can plug them into UrlTemplate directly
	Query func(reqData *RequestData) (url.Values, error) `comment:"Extra query string when requesting API, like 'Since' option for jira issues collection"`
	// Some api might do pagination by http headers
	Header      func(reqData *RequestData) (http.Header, error)
	PageSize    int
	Incremental bool `comment:"Indicate this is a incremental collection, so the existing data won't get flushed"`
	ApiClient   RateLimitedApiClient
	/*
		Sometimes, we need to collect data based on previous collected data, like jira changelog, it requires
		issue_id as part of the url.
		We can mimic `stdin` design, to accept a `Input` function which produces a `Iterator`, collector
		should iterate all records, and do data-fetching for each on, either in parallel or sequential order
		UrlTemplate: "api/3/issue/{{ Input.ID }}/changelog"
	*/
	Input          Iterator
	InputRateLimit int
	/*
		For api endpoint that returns number of total pages, ApiCollector can collect pages in parallel with ease,
		or other techniques are required if this information was missing.
	*/
	GetTotalPages  func(res *http.Response, args *ApiCollectorArgs) (int, error)
	Concurrency    int
	ResponseParser func(res *http.Response) ([]json.RawMessage, error)
	AfterResponse  ApiClientAfterResponse
}

type ApiExtractor struct {
	*RawDataSubTask
	// contains filtered or unexported fields
}

type ApiExtractorArgs struct {
	RawDataSubTaskArgs
	Params    interface{}
	Extract   RawDataExtractor
	BatchSize int
}

type ApiRateLimitCalculator struct {
	UserRateLimitPerHour   int
	GlobalRateLimitPerHour int
	MaxRetry               int
	Method                 string
	ApiPath                string
	DynamicRateLimit       func(res *http.Response) (int, time.Duration, error)
}

type AsyncResponseHandler func(res *http.Response) error

type BatchSave struct {
	// contains filtered or unexported fields
}

type BatchSaveDivider struct {
	// contains filtered or unexported fields
}

type CSTTime time.Time

type CursorIterator struct {
	// contains filtered or unexported fields
}

type DataConvertHandler func(row interface{}) ([]interface{}, error)

type DataConverter struct {
	*RawDataSubTask
	// contains filtered or unexported fields
}

type DataConverterArgs struct {
	RawDataSubTaskArgs
	// Domain layer entity Id prefix, i.e. `jira:JiraIssue:1`, `github:GithubIssue`
	InputRowType reflect.Type
	// Cursor to a set of Tool Layer Records
	Input     *sql.Rows
	Convert   DataConvertHandler
	BatchSize int
}

type DateIterator struct {
	Days    int
	Current int
	// contains filtered or unexported fields
}

type DatePair struct {
	PairStartTime time.Time
	PairEndTime   time.Time
}

type DateTimeFormatItem struct {
	Matcher *regexp.Regexp
	Format  string
}

type DefaultLogger struct {
	// contains filtered or unexported fields
}

type DefaultSubTaskContext struct {
	// contains filtered or unexported fields
}

type DefaultTaskContext struct {
	// contains filtered or unexported fields
}

type Iso8601Time struct {
	// contains filtered or unexported fields
}

type Iterator interface {
	HasNext() bool
	Fetch() (interface{}, error)
	Close() error
}

type OnNewBatchSave func(rowType reflect.Type) error

type Pager struct {
	Page int
	Skip int
	Size int
}

type RateLimitedApiClient interface {
	GetAsync(path string, query url.Values, header http.Header, handler ApiAsyncCallback) error
	WaitAsync() error
	GetQps() float64
	Add(delta int)
	Done()
	SetAfterFunction(callback ApiClientAfterResponse)
}

type RawData struct {
	ID        uint64 `gorm:"primaryKey"`
	Params    string `gorm:"type:varchar(255);index"`
	Data      []byte
	Url       string
	Input     datatypes.JSON
	CreatedAt time.Time
}

type RawDataExtractor func(row *RawData) ([]interface{}, error)

type RawDataSubTask struct {
	// contains filtered or unexported fields
}

type RawDataSubTaskArgs struct {
	Ctx core.SubTaskContext

	//	Table store raw data
	Table string `comment:"Raw data table name"`

	//	This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
	//	set of data to be process, for example, we process JiraIssues by Board
	Params interface{} `comment:"To identify a set of records with same UrlTemplate, i.e. {ConnectionId, BoardId} for jira entities"`
}

type RequestData struct {
	Pager  *Pager
	Params interface{}
	Input  interface{}
}

type WorkerScheduler struct {
	// contains filtered or unexported fields
}