serverlessspark

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 10, 2026 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const SourceType string = "serverless-spark"

Variables

This section is empty.

Functions

func BatchConsoleURL 

func BatchConsoleURL(projectID, location, batchID string) string

BatchConsoleURL builds a URL to the Google Cloud Console linking to the batch summary page.

func BatchConsoleURLFromProto 

func BatchConsoleURLFromProto(batchPb *dataprocpb.Batch) (string, error)

BatchConsoleURLFromProto builds a URL to the Google Cloud Console linking to the batch summary page.

func BatchLogsURL 

func BatchLogsURL(projectID, location, batchID string, startTime, endTime time.Time) string

BatchLogsURL builds a URL to the Google Cloud Console showing Cloud Logging for the given batch and time range.

The implementation adds some buffer before and after the provided times.

func BatchLogsURLFromProto 

func BatchLogsURLFromProto(batchPb *dataprocpb.Batch) (string, error)

BatchLogsURLFromProto builds a URL to the Google Cloud Console showing Cloud Logging for the given batch and time range.

func ExtractBatchDetails 

func ExtractBatchDetails(batchName string) (projectID, location, batchID string, err error)

Extract BatchDetails extracts the project ID, location, and batch ID from a fully qualified batch name.

func ExtractSessionDetails 

func ExtractSessionDetails(sessionName string) (projectID, location, sessionID string, err error)

ExtractSessionDetails extracts the project ID, location, and session ID from a fully qualified session name.

func ExtractSessionTemplateDetails 

func ExtractSessionTemplateDetails(sessionTemplateName string) (projectID, location, sessionTemplateID string, err error)

ExtractSessionTemplateDetails extracts the project ID, location, and session template ID from a fully qualified sessionTemplateName.

func SessionConsoleURL 

func SessionConsoleURL(projectID, location, sessionID string) string

SessionConsoleURL builds a URL to the Google Cloud Console linking to the session summary page.

func SessionConsoleURLFromProto 

func SessionConsoleURLFromProto(sessionPb *dataprocpb.Session) (string, error)

SessionConsoleURLFromProto builds a URL to the Google Cloud Console linking to the session summary page.

func SessionLogsURL 

func SessionLogsURL(projectID, location, sessionID string, startTime, endTime time.Time) string

SessionLogsURL builds a URL to the Google Cloud Console showing Cloud Logging for the given session and time range.

func SessionLogsURLFromProto 

func SessionLogsURLFromProto(sessionPb *dataprocpb.Session) (string, error)

SessionLogsURLFromProto builds a URL to the Google Cloud Console showing Cloud Logging for the given session and time range.

Types

type Batch 

type Batch struct {
	Name       string `json:"name"`
	UUID       string `json:"uuid"`
	State      string `json:"state"`
	Creator    string `json:"creator"`
	CreateTime string `json:"createTime"`
	Operation  string `json:"operation"`
	ConsoleURL string `json:"consoleUrl"`
	LogsURL    string `json:"logsUrl"`
}

Batch represents a single batch job.

func ToBatches 

func ToBatches(batchPbs []*dataprocpb.Batch) ([]Batch, error)

ToBatches converts a slice of protobuf Batch messages to a slice of Batch structs.

type Config 

type Config struct {
	Name     string `yaml:"name" validate:"required"`
	Type     string `yaml:"type" validate:"required"`
	Project  string `yaml:"project" validate:"required"`
	Location string `yaml:"location" validate:"required"`
}

func (Config) Initialize 

func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error)

func (Config) SourceConfigType 

func (r Config) SourceConfigType() string

type ListBatchesResponse 

type ListBatchesResponse struct {
	Batches       []Batch `json:"batches"`
	NextPageToken string  `json:"nextPageToken"`
}

ListBatchesResponse is the response from the list batches API.

type ListSessionsResponse 

type ListSessionsResponse struct {
	Sessions      []Session `json:"sessions"`
	NextPageToken string    `json:"nextPageToken"`
}

ListSessionsResponse is the response from the list sessions API.

type Session 

type Session struct {
	Name       string `json:"name"`
	UUID       string `json:"uuid"`
	State      string `json:"state"`
	Creator    string `json:"creator"`
	CreateTime string `json:"createTime"`
	ConsoleURL string `json:"consoleUrl"`
	LogsURL    string `json:"logsUrl"`
}

Session represents a single session job.

func ToSessions 

func ToSessions(sessionPbs []*dataprocpb.Session) ([]Session, error)

ToSessions converts a slice of protobuf Session messages to a slice of Session structs.

type SessionTemplate 

type SessionTemplate struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Creator     string `json:"creator"`
	CreateTime  string `json:"createTime"`
}

SessionTemplate represents a single session template.

func ToSessionTemplates 

func ToSessionTemplates(sessionTemplatePbs []*dataprocpb.SessionTemplate) ([]SessionTemplate, error)

ToSessionTemplates converts a slice of protobuf SessionTemplate messages to a slice of SessionTemplate structs.

type Source 

type Source struct {
	Config
	BatchClient           *dataproc.BatchControllerClient
	SessionTemplateClient *dataproc.SessionTemplateControllerClient
	OpsClient             *longrunning.OperationsClient
	SessionClient         *dataproc.SessionControllerClient
}

func (*Source) CancelOperation 

func (s *Source) CancelOperation(ctx context.Context, operation string) (any, error)

func (*Source) Close 

func (s *Source) Close() error

func (*Source) CreateBatch 

func (s *Source) CreateBatch(ctx context.Context, batch *dataprocpb.Batch) (map[string]any, error)

func (*Source) GetBatch 

func (s *Source) GetBatch(ctx context.Context, name string) (map[string]any, error)

func (*Source) GetBatchControllerClient 

func (s *Source) GetBatchControllerClient() *dataproc.BatchControllerClient

func (*Source) GetLocation 

func (s *Source) GetLocation() string

func (*Source) GetOperationsClient 

func (s *Source) GetOperationsClient(ctx context.Context) (*longrunning.OperationsClient, error)

func (*Source) GetProject 

func (s *Source) GetProject() string

func (*Source) GetSession 

func (s *Source) GetSession(ctx context.Context, name string) (map[string]any, error)

func (*Source) GetSessionControllerClient 

func (s *Source) GetSessionControllerClient() *dataproc.SessionControllerClient

func (*Source) GetSessionTemplate 

func (s *Source) GetSessionTemplate(ctx context.Context, name string) (map[string]any, error)

func (*Source) GetSessionTemplateControllerClient 

func (s *Source) GetSessionTemplateControllerClient() *dataproc.SessionTemplateControllerClient

func (*Source) ListBatches 

func (s *Source) ListBatches(ctx context.Context, ps *int, pt, filter string) (any, error)

func (*Source) ListSessions 

func (s *Source) ListSessions(ctx context.Context, ps *int, pt, filter string) (any, error)

func (*Source) SourceType 

func (s *Source) SourceType() string

func (*Source) ToConfig 

func (s *Source) ToConfig() sources.SourceConfig

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.