Documentation
¶
Index ¶
- Constants
- Variables
- func DescendArgs(args interface{}) (descended interface{}, err error)
- func FormatHookOutputPath(runID, hookRunID string) string
- func FormatRunManifestOutputPath(runID string) string
- func LuaRun(l *lua.State, code, name string) error
- func NewHookRunID(actionIdx, hookIdx int) string
- func RunByBranchPath(repoID, branchID, runID string) []byte
- func RunByCommitPath(repoID, commitID, runID string) []byte
- func RunPath(repoID, runID string) []byte
- func TasksPath(repoID, runID string) string
- type Action
- type ActionHook
- type ActionOn
- type Airflow
- type DagRunReq
- type DecreasingIDGenerator
- type EventInfo
- type Hook
- func NewAirflowHook(h ActionHook, action *Action, endpoint *http.Server) (Hook, error)
- func NewHook(h ActionHook, a *Action, e *http.Server) (Hook, error)
- func NewLuaHook(h ActionHook, action *Action, e *http.Server) (Hook, error)
- func NewWebhook(h ActionHook, action *Action, e *http.Server) (Hook, error)
- type HookBase
- type HookOutputWriter
- type HookType
- type IDGenerator
- type IncreasingIDGenerator
- type KVRunResultIterator
- type KVStore
- func (kvs *KVStore) GetRunResult(ctx context.Context, repositoryID string, runID string) (*RunResult, error)
- func (kvs *KVStore) GetTaskResult(ctx context.Context, repositoryID string, runID string, hookRunID string) (*TaskResult, error)
- func (kvs *KVStore) ListRunResults(ctx context.Context, repositoryID string, branchID, commitID string, ...) (RunResultIterator, error)
- func (kvs *KVStore) ListRunTaskResults(ctx context.Context, repositoryID string, runID string, after string) (TaskResultIterator, error)
- func (kvs *KVStore) UpdateCommitID(ctx context.Context, repositoryID string, runID string, commitID string) (*RunManifest, error)
- type KVTaskResultIterator
- type LuaHook
- type MatchSpec
- type NewHookFunc
- type OutputWriter
- type Properties
- type RunManifest
- type RunResult
- type RunResultData
- func (*RunResultData) Descriptor() ([]byte, []int)deprecated
- func (x *RunResultData) GetBranchId() string
- func (x *RunResultData) GetCommitId() string
- func (x *RunResultData) GetEndTime() *timestamppb.Timestamp
- func (x *RunResultData) GetEventType() string
- func (x *RunResultData) GetPassed() bool
- func (x *RunResultData) GetRunId() string
- func (x *RunResultData) GetSourceRef() string
- func (x *RunResultData) GetStartTime() *timestamppb.Timestamp
- func (*RunResultData) ProtoMessage()
- func (x *RunResultData) ProtoReflect() protoreflect.Message
- func (x *RunResultData) Reset()
- func (x *RunResultData) String() string
- type RunResultIterator
- type SecureString
- type Service
- type Source
- type Store
- type StoreService
- func (s *StoreService) GetRunResult(ctx context.Context, repositoryID string, runID string) (*RunResult, error)
- func (s *StoreService) GetTaskResult(ctx context.Context, repositoryID string, runID string, hookRunID string) (*TaskResult, error)
- func (s *StoreService) ListRunResults(ctx context.Context, repositoryID string, branchID, commitID string, ...) (RunResultIterator, error)
- func (s *StoreService) ListRunTaskResults(ctx context.Context, repositoryID string, runID string, after string) (TaskResultIterator, error)
- func (s *StoreService) NewRunID() string
- func (s *StoreService) PostCommitHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PostCreateBranchHook(ctx context.Context, record graveler.HookRecord)
- func (s *StoreService) PostCreateTagHook(ctx context.Context, record graveler.HookRecord)
- func (s *StoreService) PostDeleteBranchHook(ctx context.Context, record graveler.HookRecord)
- func (s *StoreService) PostDeleteTagHook(ctx context.Context, record graveler.HookRecord)
- func (s *StoreService) PostMergeHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PreCommitHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PreCreateBranchHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PreCreateTagHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PreDeleteBranchHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PreDeleteTagHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) PreMergeHook(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) Run(ctx context.Context, record graveler.HookRecord) error
- func (s *StoreService) SetEndpoint(h *http.Server)
- func (s *StoreService) Stop()
- func (s *StoreService) UpdateCommitID(ctx context.Context, repositoryID string, storageNamespace string, ...) error
- type Task
- type TaskResult
- type TaskResultData
- func (*TaskResultData) Descriptor() ([]byte, []int)deprecated
- func (x *TaskResultData) GetActionName() string
- func (x *TaskResultData) GetEndTime() *timestamppb.Timestamp
- func (x *TaskResultData) GetHookId() string
- func (x *TaskResultData) GetHookRunId() string
- func (x *TaskResultData) GetPassed() bool
- func (x *TaskResultData) GetRunId() string
- func (x *TaskResultData) GetStartTime() *timestamppb.Timestamp
- func (*TaskResultData) ProtoMessage()
- func (x *TaskResultData) ProtoReflect() protoreflect.Message
- func (x *TaskResultData) Reset()
- func (x *TaskResultData) String() string
- type TaskResultIterator
- type Webhook
Constants ¶
const ( LogOutputExtension = ".log" LogOutputLocation = "_lakefs/actions/log" )
const (
HeadersPropertyKey = "headers"
)
const (
PartitionKey = "actions"
)
Variables ¶
var ( ErrInvalidAction = errors.New("invalid action") ErrInvalidEventParameter = errors.New("invalid event parameter") )
var ( ErrNotFound = errors.New("not found") ErrNilValue = errors.New("nil value") )
var ErrParamConflict = errors.New("parameters conflict")
var ErrUnknownHookType = errors.New("unknown hook type")
var File_actions_proto protoreflect.FileDescriptor
Functions ¶
func DescendArgs ¶ added in v0.87.0
func DescendArgs(args interface{}) (descended interface{}, err error)
func FormatHookOutputPath ¶
func NewHookRunID ¶
func RunByBranchPath ¶ added in v0.67.0
func RunByCommitPath ¶ added in v0.67.0
Types ¶
type Action ¶
type Action struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
On map[graveler.EventType]*ActionOn `yaml:"on"`
Hooks []ActionHook `yaml:"hooks"`
}
func LoadActions ¶
func ParseAction ¶
ParseAction helper function to read, parse and validate Action from a reader
type ActionHook ¶
type ActionHook struct {
ID string `yaml:"id"`
Type HookType `yaml:"type"`
Description string `yaml:"description"`
Properties Properties `yaml:"properties"`
}
type Airflow ¶ added in v0.47.0
type DecreasingIDGenerator ¶ added in v0.67.0
type DecreasingIDGenerator struct{}
DecreasingIDGenerator creates IDs that are decreasing with time
func (*DecreasingIDGenerator) NewRunID ¶ added in v0.67.0
func (gen *DecreasingIDGenerator) NewRunID() string
type EventInfo ¶ added in v0.47.0
type EventInfo struct {
EventType string `json:"event_type"`
EventTime string `json:"event_time"`
ActionName string `json:"action_name"`
HookID string `json:"hook_id"`
RepositoryID string `json:"repository_id"`
BranchID string `json:"branch_id,omitempty"`
SourceRef string `json:"source_ref,omitempty"`
TagID string `json:"tag_id,omitempty"`
CommitID string `json:"commit_id,omitempty"`
CommitMessage string `json:"commit_message,omitempty"`
Committer string `json:"committer,omitempty"`
CommitMetadata map[string]string `json:"commit_metadata,omitempty"`
}
type Hook ¶
type Hook interface {
Run(ctx context.Context, record graveler.HookRecord, buf *bytes.Buffer) error
}
Hook is the abstraction of the basic user-configured runnable building-stone
func NewAirflowHook ¶ added in v0.47.0
func NewLuaHook ¶ added in v0.87.0
func NewWebhook ¶
type HookOutputWriter ¶
type HookOutputWriter struct {
StorageNamespace string
RunID string
HookRunID string
ActionName string
HookID string
Writer OutputWriter
}
func (*HookOutputWriter) OutputWrite ¶
type IDGenerator ¶ added in v0.67.0
type IDGenerator interface {
// NewRunID creates IDs for Runs.
NewRunID() string
}
type IncreasingIDGenerator ¶ added in v0.67.0
type IncreasingIDGenerator struct{}
IncreasingIDGenerator creates IDs that are increasing with time
func (*IncreasingIDGenerator) NewRunID ¶ added in v0.67.0
func (gen *IncreasingIDGenerator) NewRunID() string
type KVRunResultIterator ¶ added in v0.67.0
type KVRunResultIterator struct {
// contains filtered or unexported fields
}
func NewKVRunResultIterator ¶ added in v0.67.0
func NewKVRunResultIterator(ctx context.Context, store kv.StoreMessage, repositoryID, branchID, commitID, after string) (*KVRunResultIterator, error)
NewKVRunResultIterator returns a new iterator over actions run results 'after' determines the runID which we should start the scan from, used for pagination
func (*KVRunResultIterator) Close ¶ added in v0.67.0
func (i *KVRunResultIterator) Close()
func (*KVRunResultIterator) Err ¶ added in v0.67.0
func (i *KVRunResultIterator) Err() error
func (*KVRunResultIterator) Next ¶ added in v0.67.0
func (i *KVRunResultIterator) Next() bool
func (*KVRunResultIterator) Value ¶ added in v0.67.0
func (i *KVRunResultIterator) Value() *RunResult
type KVStore ¶ added in v0.67.0
type KVStore struct {
// contains filtered or unexported fields
}
func (*KVStore) GetRunResult ¶ added in v0.67.0
func (*KVStore) GetTaskResult ¶ added in v0.67.0
func (*KVStore) ListRunResults ¶ added in v0.67.0
func (*KVStore) ListRunTaskResults ¶ added in v0.67.0
func (*KVStore) UpdateCommitID ¶ added in v0.67.0
func (kvs *KVStore) UpdateCommitID(ctx context.Context, repositoryID string, runID string, commitID string) (*RunManifest, error)
UpdateCommitID assume record is a post event, we use the PreRunID to update the commit_id and save the run manifest again
type KVTaskResultIterator ¶ added in v0.67.0
type KVTaskResultIterator struct {
// contains filtered or unexported fields
}
func NewKVTaskResultIterator ¶ added in v0.67.0
func NewKVTaskResultIterator(ctx context.Context, store kv.StoreMessage, repositoryID, runID, after string) (*KVTaskResultIterator, error)
NewKVTaskResultIterator returns a new iterator over actions task results of a specific run 'after' determines the hook run ID which we should start the scan from, used for pagination
func (*KVTaskResultIterator) Close ¶ added in v0.67.0
func (i *KVTaskResultIterator) Close()
func (*KVTaskResultIterator) Err ¶ added in v0.67.0
func (i *KVTaskResultIterator) Err() error
func (*KVTaskResultIterator) Next ¶ added in v0.67.0
func (i *KVTaskResultIterator) Next() bool
func (*KVTaskResultIterator) Value ¶ added in v0.67.0
func (i *KVTaskResultIterator) Value() *TaskResult
type NewHookFunc ¶
type OutputWriter ¶
type Properties ¶ added in v0.47.0
type Properties map[string]interface{}
type RunManifest ¶
type RunManifest struct {
Run RunResult `json:"run"`
HooksRun []TaskResult `json:"hooks,omitempty"`
}
type RunResult ¶
type RunResult struct {
RunID string `db:"run_id" json:"run_id"`
BranchID string `db:"branch_id" json:"branch_id"`
SourceRef string `db:"source_ref" json:"source_ref"`
EventType string `db:"event_type" json:"event_type"`
CommitID string `db:"commit_id" json:"commit_id,omitempty"`
StartTime time.Time `db:"start_time" json:"start_time"`
EndTime time.Time `db:"end_time" json:"end_time"`
Passed bool `db:"passed" json:"passed"`
}
func RunResultFromProto ¶ added in v0.67.0
func RunResultFromProto(pb *RunResultData) *RunResult
type RunResultData ¶ added in v0.67.0
type RunResultData struct {
RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"`
BranchId string `protobuf:"bytes,2,opt,name=branch_id,json=branchId,proto3" json:"branch_id,omitempty"`
CommitId string `protobuf:"bytes,3,opt,name=commit_id,json=commitId,proto3" json:"commit_id,omitempty"`
SourceRef string `protobuf:"bytes,4,opt,name=source_ref,json=sourceRef,proto3" json:"source_ref,omitempty"`
EventType string `protobuf:"bytes,5,opt,name=event_type,json=eventType,proto3" json:"event_type,omitempty"`
StartTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"`
EndTime *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"`
Passed bool `protobuf:"varint,8,opt,name=passed,proto3" json:"passed,omitempty"`
// contains filtered or unexported fields
}
message data model for RunResult struct
func (*RunResultData) Descriptor
deprecated
added in
v0.67.0
func (*RunResultData) Descriptor() ([]byte, []int)
Deprecated: Use RunResultData.ProtoReflect.Descriptor instead.
func (*RunResultData) GetBranchId ¶ added in v0.67.0
func (x *RunResultData) GetBranchId() string
func (*RunResultData) GetCommitId ¶ added in v0.67.0
func (x *RunResultData) GetCommitId() string
func (*RunResultData) GetEndTime ¶ added in v0.67.0
func (x *RunResultData) GetEndTime() *timestamppb.Timestamp
func (*RunResultData) GetEventType ¶ added in v0.67.0
func (x *RunResultData) GetEventType() string
func (*RunResultData) GetPassed ¶ added in v0.67.0
func (x *RunResultData) GetPassed() bool
func (*RunResultData) GetRunId ¶ added in v0.67.0
func (x *RunResultData) GetRunId() string
func (*RunResultData) GetSourceRef ¶ added in v0.67.0
func (x *RunResultData) GetSourceRef() string
func (*RunResultData) GetStartTime ¶ added in v0.67.0
func (x *RunResultData) GetStartTime() *timestamppb.Timestamp
func (*RunResultData) ProtoMessage ¶ added in v0.67.0
func (*RunResultData) ProtoMessage()
func (*RunResultData) ProtoReflect ¶ added in v0.67.0
func (x *RunResultData) ProtoReflect() protoreflect.Message
func (*RunResultData) Reset ¶ added in v0.67.0
func (x *RunResultData) Reset()
func (*RunResultData) String ¶ added in v0.67.0
func (x *RunResultData) String() string
type RunResultIterator ¶
type SecureString ¶ added in v0.48.0
type SecureString struct {
// contains filtered or unexported fields
}
SecureString is a string that may be populated from an environment variable. If constructed with a string of the form {{ ENV.EXAMPLE_VARIABLE }}, the value is populated from EXAMPLE_VARIABLE and is considered a secret. Otherwise the value is taken from the string as-is, and is not considered a secret.
func NewSecureString ¶ added in v0.48.0
func NewSecureString(s string) (SecureString, error)
NewSecureString creates a new SecureString, reading env var if needed.
func (SecureString) String ¶ added in v0.48.0
func (s SecureString) String() string
Returns the string for non-secrets, or asterisks otherwise.
type Service ¶
type Service interface {
Stop()
Run(ctx context.Context, record graveler.HookRecord) error
UpdateCommitID(ctx context.Context, repositoryID string, storageNamespace string, runID string, commitID string) error
GetRunResult(ctx context.Context, repositoryID string, runID string) (*RunResult, error)
GetTaskResult(ctx context.Context, repositoryID string, runID string, hookRunID string) (*TaskResult, error)
ListRunResults(ctx context.Context, repositoryID string, branchID, commitID string, after string) (RunResultIterator, error)
ListRunTaskResults(ctx context.Context, repositoryID string, runID string, after string) (TaskResultIterator, error)
graveler.HooksHandler
}
type Store ¶ added in v0.67.0
type Store interface {
// UpdateCommitID will update an already stored run with the commit results
UpdateCommitID(ctx context.Context, repositoryID string, runID string, commitID string) (*RunManifest, error)
GetRunResult(ctx context.Context, repositoryID string, runID string) (*RunResult, error)
GetTaskResult(ctx context.Context, repositoryID string, runID string, hookRunID string) (*TaskResult, error)
ListRunResults(ctx context.Context, repositoryID string, branchID, commitID string, after string) (RunResultIterator, error)
ListRunTaskResults(ctx context.Context, repositoryID string, runID string, after string) (TaskResultIterator, error)
// contains filtered or unexported methods
}
Store is an abstraction layer for operating with a concrete postgres DB or a KV store abstraction.
func NewActionsKVStore ¶ added in v0.67.0
func NewActionsKVStore(store kv.StoreMessage) Store
type StoreService ¶ added in v0.67.0
type StoreService struct {
Store Store
Source Source
Writer OutputWriter
// contains filtered or unexported fields
}
StoreService is an implementation of actions.Service that saves the run data to the blockstore and to the actions.Store (which is a fancy name for a DB - kv style or postgres directly)
func NewService ¶
func NewService(ctx context.Context, store Store, source Source, writer OutputWriter, idGen IDGenerator, stats stats.Collector, runHooks bool) *StoreService
func (*StoreService) GetRunResult ¶ added in v0.67.0
func (*StoreService) GetTaskResult ¶ added in v0.67.0
func (s *StoreService) GetTaskResult(ctx context.Context, repositoryID string, runID string, hookRunID string) (*TaskResult, error)
func (*StoreService) ListRunResults ¶ added in v0.67.0
func (s *StoreService) ListRunResults(ctx context.Context, repositoryID string, branchID, commitID string, after string) (RunResultIterator, error)
func (*StoreService) ListRunTaskResults ¶ added in v0.67.0
func (s *StoreService) ListRunTaskResults(ctx context.Context, repositoryID string, runID string, after string) (TaskResultIterator, error)
func (*StoreService) NewRunID ¶ added in v0.67.0
func (s *StoreService) NewRunID() string
func (*StoreService) PostCommitHook ¶ added in v0.67.0
func (s *StoreService) PostCommitHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PostCreateBranchHook ¶ added in v0.67.0
func (s *StoreService) PostCreateBranchHook(ctx context.Context, record graveler.HookRecord)
func (*StoreService) PostCreateTagHook ¶ added in v0.67.0
func (s *StoreService) PostCreateTagHook(ctx context.Context, record graveler.HookRecord)
func (*StoreService) PostDeleteBranchHook ¶ added in v0.67.0
func (s *StoreService) PostDeleteBranchHook(ctx context.Context, record graveler.HookRecord)
func (*StoreService) PostDeleteTagHook ¶ added in v0.67.0
func (s *StoreService) PostDeleteTagHook(ctx context.Context, record graveler.HookRecord)
func (*StoreService) PostMergeHook ¶ added in v0.67.0
func (s *StoreService) PostMergeHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PreCommitHook ¶ added in v0.67.0
func (s *StoreService) PreCommitHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PreCreateBranchHook ¶ added in v0.67.0
func (s *StoreService) PreCreateBranchHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PreCreateTagHook ¶ added in v0.67.0
func (s *StoreService) PreCreateTagHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PreDeleteBranchHook ¶ added in v0.67.0
func (s *StoreService) PreDeleteBranchHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PreDeleteTagHook ¶ added in v0.67.0
func (s *StoreService) PreDeleteTagHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) PreMergeHook ¶ added in v0.67.0
func (s *StoreService) PreMergeHook(ctx context.Context, record graveler.HookRecord) error
func (*StoreService) Run ¶ added in v0.67.0
func (s *StoreService) Run(ctx context.Context, record graveler.HookRecord) error
Run load and run actions based on the event information
func (*StoreService) SetEndpoint ¶ added in v0.87.0
func (s *StoreService) SetEndpoint(h *http.Server)
func (*StoreService) Stop ¶ added in v0.67.0
func (s *StoreService) Stop()
func (*StoreService) UpdateCommitID ¶ added in v0.67.0
func (s *StoreService) UpdateCommitID(ctx context.Context, repositoryID string, storageNamespace string, runID string, commitID string) error
UpdateCommitID assume record is a post event, we use the PreRunID to update the commit_id and save the run manifest again
type TaskResult ¶
type TaskResult struct {
RunID string `db:"run_id" json:"run_id"`
HookRunID string `db:"hook_run_id" json:"hook_run_id"`
HookID string `db:"hook_id" json:"hook_id"`
ActionName string `db:"action_name" json:"action_name"`
StartTime time.Time `db:"start_time" json:"start_time"`
EndTime time.Time `db:"end_time" json:"end_time"`
Passed bool `db:"passed" json:"passed"`
}
func (*TaskResult) LogPath ¶
func (r *TaskResult) LogPath() string
type TaskResultData ¶ added in v0.67.0
type TaskResultData struct {
RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"`
HookRunId string `protobuf:"bytes,2,opt,name=hook_run_id,json=hookRunId,proto3" json:"hook_run_id,omitempty"`
HookId string `protobuf:"bytes,3,opt,name=hook_id,json=hookId,proto3" json:"hook_id,omitempty"`
ActionName string `protobuf:"bytes,4,opt,name=action_name,json=actionName,proto3" json:"action_name,omitempty"`
StartTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"`
EndTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"`
Passed bool `protobuf:"varint,9,opt,name=passed,proto3" json:"passed,omitempty"`
// contains filtered or unexported fields
}
message data model for TaskResult struct
func (*TaskResultData) Descriptor
deprecated
added in
v0.67.0
func (*TaskResultData) Descriptor() ([]byte, []int)
Deprecated: Use TaskResultData.ProtoReflect.Descriptor instead.
func (*TaskResultData) GetActionName ¶ added in v0.67.0
func (x *TaskResultData) GetActionName() string
func (*TaskResultData) GetEndTime ¶ added in v0.67.0
func (x *TaskResultData) GetEndTime() *timestamppb.Timestamp
func (*TaskResultData) GetHookId ¶ added in v0.67.0
func (x *TaskResultData) GetHookId() string
func (*TaskResultData) GetHookRunId ¶ added in v0.67.0
func (x *TaskResultData) GetHookRunId() string
func (*TaskResultData) GetPassed ¶ added in v0.67.0
func (x *TaskResultData) GetPassed() bool
func (*TaskResultData) GetRunId ¶ added in v0.67.0
func (x *TaskResultData) GetRunId() string
func (*TaskResultData) GetStartTime ¶ added in v0.67.0
func (x *TaskResultData) GetStartTime() *timestamppb.Timestamp
func (*TaskResultData) ProtoMessage ¶ added in v0.67.0
func (*TaskResultData) ProtoMessage()
func (*TaskResultData) ProtoReflect ¶ added in v0.67.0
func (x *TaskResultData) ProtoReflect() protoreflect.Message
func (*TaskResultData) Reset ¶ added in v0.67.0
func (x *TaskResultData) Reset()
func (*TaskResultData) String ¶ added in v0.67.0
func (x *TaskResultData) String() string
type TaskResultIterator ¶
type TaskResultIterator interface {
Next() bool
Value() *TaskResult
Err() error
Close()
}