emr

package
v1.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 3, 2026 License: MIT Imports: 26 Imported by: 0

README

EMR

Parity grade: A · SDK aws-sdk-go-v2/service/emr@v1.64.0 · last audited 2026-07-25 (44f89c945)

Coverage

Metric Value
Operations audited 66 (65 ok, 1 partial)
Known gaps none
Deferred items 0
Resource leaks clean

More

Documentation

Index

Constants

View Source 
const (
	StateWaiting              = "WAITING"
	StateTerminated           = "TERMINATED"
	StateTerminatedWithErrors = "TERMINATED_WITH_ERRORS"

	// StateRunning is the real ClusterState value StartSession's doc allows
	// sessions to start against, alongside StateWaiting. This backend's own
	// cluster state machine never produces RUNNING -- clusters are created
	// directly in WAITING (see buildNewCluster) and go straight from WAITING
	// to TERMINATED (see terminateSingle), with no simulated
	// STARTING/BOOTSTRAPPING/RUNNING/TERMINATING in between -- so in
	// practice only StateWaiting ever passes the sessionCanStart check
	// below. StateRunning is included for correctness against the real API
	// and forward-compatibility (e.g. a hand-seeded test cluster).
	StateRunning = "RUNNING"

	StepStatePending   = "PENDING"
	StepStateCompleted = "COMPLETED"
	StepStateCancelled = "CANCELLED"

	// SessionStateSubmitted and its siblings below mirror the real
	// SessionState enum (aws-sdk-go-v2/service/emr/types/enums.go) verbatim:
	// SUBMITTED, STARTING, STARTED, IDLE, BUSY, TERMINATING, TERMINATED,
	// FAILED. This backend only ever drives two of them -- see sessions.go's
	// package doc comment for the full state-model rationale.
	SessionStateSubmitted   = "SUBMITTED"
	SessionStateStarting    = "STARTING"
	SessionStateStarted     = "STARTED"
	SessionStateIdle        = "IDLE"
	SessionStateBusy        = "BUSY"
	SessionStateTerminating = "TERMINATING"
	SessionStateTerminated  = "TERMINATED"
	SessionStateFailed      = "FAILED"
)
View Source 
const (
	NotebookStatusRunning  = "RUNNING"
	NotebookStatusStopping = "STOPPING"
	NotebookStatusStopped  = "STOPPED"
	NotebookStatusFinished = "FINISHED"
)

NotebookExecutionStatus values for a notebook execution.

Variables

View Source 
var (
	ErrNotFound      = awserr.New("ClientException", awserr.ErrNotFound)
	ErrAlreadyExists = awserr.New("ClientException", awserr.ErrAlreadyExists)
)
View Source 
var ErrNilAppContext = errors.New("EMR provider: nil AppContext")

ErrNilAppContext is returned when Init is called with a nil AppContext.

View Source 
var ErrValidation = awserr.New(
	"ValidationException: required field is missing",
	awserr.ErrInvalidParameter,
)

Functions

This section is empty.

Types

type Application 

type Application struct {
	Name    string `json:"Name"`
	Version string `json:"Version,omitempty"`
}

Application represents an EMR application bundled in a cluster.

type AutoScalingConstraints 

type AutoScalingConstraints struct {
	MinCapacity int `json:"MinCapacity"`
	MaxCapacity int `json:"MaxCapacity"`
}

AutoScalingConstraints defines capacity bounds for an auto-scaling policy.

type AutoScalingPolicyDetail 

type AutoScalingPolicyDetail struct {
	Status      map[string]string      `json:"Status"`
	Rules       []ScalingRule          `json:"Rules,omitempty"`
	Constraints AutoScalingConstraints `json:"Constraints"`
}

AutoScalingPolicyDetail is stored on an instance group after PutAutoScalingPolicy.

type AutoScalingPolicySpec 

type AutoScalingPolicySpec struct {
	Rules       []ScalingRule          `json:"Rules,omitempty"`
	Constraints AutoScalingConstraints `json:"Constraints"`
}

AutoScalingPolicySpec is used as input to PutAutoScalingPolicy.

type AutoTerminationPolicy 

type AutoTerminationPolicy struct {
	IdleTimeout int64 `json:"IdleTimeout"`
}

AutoTerminationPolicy defines the auto-termination idle timeout for a cluster.

type BlockPublicAccessConfiguration 

type BlockPublicAccessConfiguration struct {
	PermittedPublicSecurityGroupRuleRanges []PortRange `json:"PermittedPublicSecurityGroupRuleRanges,omitempty"`
	BlockPublicSecurityGroupRules          bool        `json:"BlockPublicSecurityGroupRules"`
	// contains filtered or unexported fields
}

BlockPublicAccessConfiguration is the account-level block-public-access config.

type BootstrapActionConfig 

type BootstrapActionConfig struct {
	Name                  string                `json:"Name"`
	ScriptBootstrapAction BootstrapActionScript `json:"ScriptBootstrapAction"`
}

BootstrapActionConfig is the full bootstrap action specification used in RunJobFlow input.

type BootstrapActionScript 

type BootstrapActionScript struct {
	Path string   `json:"Path"`
	Args []string `json:"Args,omitempty"`
}

BootstrapActionScript holds the script path and arguments for a bootstrap action.

type CancelStepsInfo 

type CancelStepsInfo struct {
	StepID string `json:"StepId"`
	Status string `json:"Status"`
	Reason string `json:"Reason,omitempty"`
}

CancelStepsInfo represents the result of cancelling a single step.

type CloudWatchAlarmDefinition 

type CloudWatchAlarmDefinition struct {
	ComparisonOperator string  `json:"ComparisonOperator"`
	MetricName         string  `json:"MetricName"`
	Namespace          string  `json:"Namespace,omitempty"`
	Statistic          string  `json:"Statistic,omitempty"`
	Unit               string  `json:"Unit,omitempty"`
	EvaluationPeriods  int     `json:"EvaluationPeriods"`
	Period             int     `json:"Period"`
	Threshold          float64 `json:"Threshold"`
}

CloudWatchAlarmDefinition is the CloudWatch alarm that triggers scaling.

type Cluster 

type Cluster struct {
	TerminatedAt          time.Time              `json:"TerminatedAt,omitzero"`
	Ec2InstanceAttributes *EC2InstanceAttributes `json:"Ec2InstanceAttributes"`
	KerberosAttributes    *KerberosAttributes    `json:"KerberosAttributes,omitempty"`

	Status                 ClusterStatus `json:"Status"`
	ScaleDownBehavior      string        `json:"ScaleDownBehavior,omitempty"`
	ID                     string        `json:"Id"`
	ARN                    string        `json:"ClusterArn"`
	ReleaseLabel           string        `json:"ReleaseLabel"`
	OSReleaseLabel         string        `json:"OSReleaseLabel,omitempty"`
	LogURI                 string        `json:"LogUri,omitempty"`
	ServiceRole            string        `json:"ServiceRole,omitempty"`
	AutoScalingRole        string        `json:"AutoScalingRole,omitempty"`
	Name                   string        `json:"Name"`
	SecurityConfiguration  string        `json:"SecurityConfiguration,omitempty"`
	CustomAmiID            string        `json:"CustomAmiId,omitempty"`
	InstanceCollectionType string        `json:"InstanceCollectionType,omitempty"`

	Tags            []Tag                  `json:"Tags"`
	Applications    []Application          `json:"Applications,omitempty"`
	Configurations  []Configuration        `json:"Configurations,omitempty"`
	PlacementGroups []PlacementGroupConfig `json:"PlacementGroups,omitempty"`

	StepConcurrencyLevel        int  `json:"StepConcurrencyLevel,omitempty"`
	EbsRootVolumeSize           int  `json:"EbsRootVolumeSize,omitempty"`
	EbsRootVolumeIops           int  `json:"EbsRootVolumeIops,omitempty"`
	EbsRootVolumeThroughput     int  `json:"EbsRootVolumeThroughput,omitempty"`
	UnhealthyNodeReplacement    bool `json:"UnhealthyNodeReplacement"`
	KeepJobFlowAliveWhenNoSteps bool `json:"KeepJobFlowAliveWhenNoSteps"`
	TerminationProtected        bool `json:"TerminationProtected"`
	VisibleToAllUsers           bool `json:"VisibleToAllUsers"`
	// AutoTerminate is the real API's inverse of KeepJobFlowAliveWhenNoSteps:
	// true means the cluster terminates after completing all steps.
	AutoTerminate bool `json:"AutoTerminate"`
	// contains filtered or unexported fields
}

Cluster represents an EMR cluster.

type ClusterInstance 

type ClusterInstance struct {
	ID               string                `json:"Id"`
	Ec2InstanceID    string                `json:"Ec2InstanceId"`
	PrivateDNSName   string                `json:"PrivateDnsName"`
	PublicDNSName    string                `json:"PublicDnsName,omitempty"`
	PrivateIPAddress string                `json:"PrivateIpAddress,omitempty"`
	InstanceGroupID  string                `json:"InstanceGroupId,omitempty"`
	InstanceFleetID  string                `json:"InstanceFleetId,omitempty"`
	Market           string                `json:"Market"`
	InstanceType     string                `json:"InstanceType"`
	Status           ClusterInstanceStatus `json:"Status"`
}

ClusterInstance represents a single EC2 instance in a cluster.

type ClusterInstanceStatus 

type ClusterInstanceStatus struct {
	State string `json:"State"`
}

ClusterInstanceStatus holds the state of a ClusterInstance.

type ClusterStatus 

type ClusterStatus struct {
	StateChangeReason map[string]any `json:"StateChangeReason,omitempty"`
	Timeline          map[string]any `json:"Timeline,omitempty"`
	State             string         `json:"State"`
}

ClusterStatus holds the status fields for a Cluster.

type ClusterSummary 

type ClusterSummary struct {
	ID         string        `json:"Id"`
	Name       string        `json:"Name"`
	Status     ClusterStatus `json:"Status"`
	ClusterArn string        `json:"ClusterArn"`
}

ClusterSummary is a trimmed-down view used for ListClusters.

NormalizedInstanceHours is a real ClusterSummary member (aws-sdk-go-v2/service/emr/types.ClusterSummary) this backend never populates -- an honest omission (a real client sees it as nil/zero), not fabricated. OutpostArn is likewise real and omitted for the same reason. ReleaseLabel used to live here but was deleted: the real ClusterSummary has no such member at all (only Id, Name, Status, ClusterArn, NormalizedInstanceHours, OutpostArn) -- it was an invented field, not an omission.

type Command 

type Command struct {
	Name       string   `json:"Name"`
	ScriptPath string   `json:"ScriptPath"`
	Args       []string `json:"Args,omitempty"`
}

Command is the flattened representation of a bootstrap action returned by ListBootstrapActions.

type ComputeLimits 

type ComputeLimits struct {
	UnitType                     string `json:"UnitType"`
	MinimumCapacityUnits         int    `json:"MinimumCapacityUnits"`
	MaximumCapacityUnits         int    `json:"MaximumCapacityUnits"`
	MaximumOnDemandCapacityUnits int    `json:"MaximumOnDemandCapacityUnits,omitempty"`
	MaximumCoreCapacityUnits     int    `json:"MaximumCoreCapacityUnits,omitempty"`
}

ComputeLimits defines compute bounds for managed scaling.

type ConfigProvider 

type ConfigProvider interface {
	GetEMRSettings() Settings
}

ConfigProvider is a private interface to extract EMR configuration from the abstract AppContext Config.

type Configuration 

type Configuration struct {
	Classification string            `json:"Classification,omitempty"`
	Properties     map[string]string `json:"Properties,omitempty"`
	Configurations []Configuration   `json:"Configurations,omitempty"`
}

Configuration is a recursive EMR classification configuration entry.

type EC2InstanceAttributes 

type EC2InstanceAttributes struct {
	Ec2KeyName                     string   `json:"Ec2KeyName,omitempty"`
	Ec2SubnetID                    string   `json:"Ec2SubnetId,omitempty"`
	Ec2AvailabilityZone            string   `json:"Ec2AvailabilityZone,omitempty"`
	EmrManagedMasterSecurityGroup  string   `json:"EmrManagedMasterSecurityGroup,omitempty"`
	EmrManagedSlaveSecurityGroup   string   `json:"EmrManagedSlaveSecurityGroup,omitempty"`
	ServiceAccessSecurityGroup     string   `json:"ServiceAccessSecurityGroup,omitempty"`
	IamInstanceProfile             string   `json:"IamInstanceProfile,omitempty"`
	AdditionalMasterSecurityGroups []string `json:"AdditionalMasterSecurityGroups,omitempty"`
	AdditionalSlaveSecurityGroups  []string `json:"AdditionalSlaveSecurityGroups,omitempty"`
	RequestedEc2SubnetIDs          []string `json:"RequestedEc2SubnetIds,omitempty"`
}

EC2InstanceAttributes represents EC2 instance attributes for an EMR cluster.

type Handler 

type Handler struct {
	Backend *InMemoryBackend
	// contains filtered or unexported fields
}

Handler is the Echo HTTP handler for AWS EMR operations.

func NewHandler 

func NewHandler(backend *InMemoryBackend) *Handler

NewHandler creates a new EMR handler backed by backend.

func (*Handler) ChaosOperations 

func (h *Handler) ChaosOperations() []string

ChaosOperations returns all operations that can be fault-injected.

func (*Handler) ChaosRegions 

func (h *Handler) ChaosRegions() []string

ChaosRegions returns all regions this handler instance handles.

func (*Handler) ChaosServiceName 

func (h *Handler) ChaosServiceName() string

ChaosServiceName returns the lowercase AWS service name for fault rule matching.

func (*Handler) ExtractOperation 

func (h *Handler) ExtractOperation(c *echo.Context) string

ExtractOperation returns the operation name from the X-Amz-Target header.

func (*Handler) ExtractResource 

func (h *Handler) ExtractResource(c *echo.Context) string

ExtractResource extracts a resource identifier from the request body.

func (*Handler) GetSupportedOperations 

func (h *Handler) GetSupportedOperations() []string

GetSupportedOperations returns the list of supported operations.

"ListTagsForResource" is deliberately NOT listed here -- the real EMR SDK has no such operation (verified against aws-sdk-go-v2/service/emr: only AddTags/RemoveTags exist; a real client reads tags back via DescribeCluster.Tags/DescribeStudio.Tags). See the opListTagsForResource comment in buildOps below for why the route itself stays wired.

func (*Handler) Handler 

func (h *Handler) Handler() echo.HandlerFunc

Handler returns the Echo handler function for EMR requests.

func (*Handler) MatchPriority 

func (h *Handler) MatchPriority() int

MatchPriority returns the routing priority.

func (*Handler) Name 

func (h *Handler) Name() string

Name returns the service name.

func (*Handler) Reset 

func (h *Handler) Reset()

Reset clears all in-memory state from the backend.

func (*Handler) Restore 

func (h *Handler) Restore(ctx context.Context, data []byte) error

Restore implements persistence.Persistable by delegating to the backend.

func (*Handler) RouteMatcher 

func (h *Handler) RouteMatcher() service.Matcher

RouteMatcher returns a function that matches EMR requests via X-Amz-Target.

func (*Handler) Snapshot 

func (h *Handler) Snapshot(ctx context.Context) []byte

Snapshot implements persistence.Persistable by delegating to the backend.

func (*Handler) StartWorker 

func (h *Handler) StartWorker(ctx context.Context) error

StartWorker starts the background janitor if configured.

func (*Handler) WithJanitor 

func (h *Handler) WithJanitor(interval, terminatedTTL time.Duration, taskTimeout ...time.Duration) *Handler

WithJanitor attaches a background janitor to the handler.

type InMemoryBackend 

type InMemoryBackend struct {
	// contains filtered or unexported fields
}

InMemoryBackend stores EMR state in memory.

The resource collections below were previously nested by region (outer key = region, e.g. map[string]map[string]*Cluster) so that same-named resources in different regions were fully isolated. Phase 3.3 of the datalayer refactor replaces each of those with a flat *store.Table, keyed by the composite "region|id" string (see regionKey), with a companion *store.Index grouping entries by region for per-region scans -- the same region-qualified-table pattern services/neptune and services/mwaa use. The block-public-access configuration/metadata are account-level (one per region in AWS, not per-resource), so each is a *store.Table keyed directly by region with no secondary index needed. arnIndex (map[string]map[string]string) is deliberately NOT converted: store.Table requires a *V value with its own identity, but each entry here is a bare string with no identifier of its own; it remains a plain region-nested map.

func NewInMemoryBackend 

func NewInMemoryBackend(accountID, region string) *InMemoryBackend

NewInMemoryBackend creates a new InMemoryBackend.

func (*InMemoryBackend) AddClusterInternal 

func (b *InMemoryBackend) AddClusterInternal(ctx context.Context, cluster *Cluster)

AddClusterInternal seeds a cluster directly into the backend for testing.

func (*InMemoryBackend) AddInstanceFleet 

func (b *InMemoryBackend) AddInstanceFleet(
	ctx context.Context,
	clusterID string,
	spec InstanceFleetSpec,
) (*InstanceFleet, string, error)

AddInstanceFleet adds an instance fleet to an existing cluster.

func (*InMemoryBackend) AddInstanceGroups 

func (b *InMemoryBackend) AddInstanceGroups(
	ctx context.Context,
	clusterID string,
	specs []InstanceGroupSpec,
) ([]string, string, error)

AddInstanceGroups adds new instance groups to an existing cluster.

func (*InMemoryBackend) AddJobFlowSteps 

func (b *InMemoryBackend) AddJobFlowSteps(
	ctx context.Context, jobFlowID string, specs []StepSpec,
) ([]string, error)

AddJobFlowSteps adds steps to a cluster and returns their IDs.

func (*InMemoryBackend) AddPersistentAppUIInternal 

func (b *InMemoryBackend) AddPersistentAppUIInternal(ctx context.Context, ui PersistentAppUI)

AddPersistentAppUIInternal seeds a persistent app UI directly into the backend for testing.

func (*InMemoryBackend) AddSecurityConfigInternal 

func (b *InMemoryBackend) AddSecurityConfigInternal(ctx context.Context, sc SecurityConfiguration)

AddSecurityConfigInternal seeds a security configuration directly into the backend for testing.

func (*InMemoryBackend) AddStudioInternal 

func (b *InMemoryBackend) AddStudioInternal(ctx context.Context, studio Studio)

AddStudioInternal seeds a studio directly into the backend for testing.

func (*InMemoryBackend) AddTags 

func (b *InMemoryBackend) AddTags(ctx context.Context, resourceID string, tags []Tag) error

AddTags adds or updates tags on a cluster or studio identified by ARN or ID. When resourceID is an ARN the region is resolved from the ARN, otherwise the ctx region (falling back to the backend default) is used.

func (*InMemoryBackend) CancelSteps 

func (b *InMemoryBackend) CancelSteps(
	ctx context.Context,
	clusterID string,
	stepIDs []string,
) ([]*CancelStepsInfo, error)

CancelSteps cancels pending steps on a cluster.

func (*InMemoryBackend) CreatePersistentAppUI 

func (b *InMemoryBackend) CreatePersistentAppUI(
	ctx context.Context,
	targetResourceArn string,
) (*PersistentAppUI, error)

CreatePersistentAppUI creates a new persistent application user interface.

func (*InMemoryBackend) CreateSecurityConfiguration 

func (b *InMemoryBackend) CreateSecurityConfiguration(
	ctx context.Context,
	name, securityConfig string,
) (*SecurityConfiguration, error)

CreateSecurityConfiguration creates a new security configuration.

func (*InMemoryBackend) CreateStudio 

func (b *InMemoryBackend) CreateStudio(
	ctx context.Context,
	name, description, authMode, defaultS3Location, engineSGID, serviceRole, vpcID, workspaceSGID string,
	subnetIDs []string, tags []Tag,
) (*Studio, error)

CreateStudio creates a new EMR Studio.

func (*InMemoryBackend) CreateStudioSessionMapping 

func (b *InMemoryBackend) CreateStudioSessionMapping(
	ctx context.Context,
	studioID, identityType, identityID, identityName, sessionPolicyArn string,
) error

CreateStudioSessionMapping maps a user or group to an EMR Studio.

func (*InMemoryBackend) DeleteSecurityConfiguration 

func (b *InMemoryBackend) DeleteSecurityConfiguration(ctx context.Context, name string) error

DeleteSecurityConfiguration deletes a security configuration by name.

func (*InMemoryBackend) DeleteStudio 

func (b *InMemoryBackend) DeleteStudio(ctx context.Context, studioID string) error

DeleteStudio deletes an EMR Studio by ID.

func (*InMemoryBackend) DeleteStudioSessionMapping 

func (b *InMemoryBackend) DeleteStudioSessionMapping(
	ctx context.Context,
	studioID, identityType, identityID, identityName string,
) error

DeleteStudioSessionMapping removes a user or group from an EMR Studio.

func (*InMemoryBackend) DescribeCluster 

func (b *InMemoryBackend) DescribeCluster(ctx context.Context, id string) (*Cluster, error)

DescribeCluster returns a cluster by its ID.

func (*InMemoryBackend) DescribeJobFlows 

func (b *InMemoryBackend) DescribeJobFlows(
	ctx context.Context,
	ids, states []string,
	createdAfter, createdBefore *time.Time,
) []JobFlow

DescribeJobFlows translates clusters into the legacy JobFlow format.

func (*InMemoryBackend) DescribeNotebookExecution 

func (b *InMemoryBackend) DescribeNotebookExecution(ctx context.Context, id string) (*NotebookExecution, error)

DescribeNotebookExecution returns a notebook execution by ID.

func (*InMemoryBackend) DescribePersistentAppUI 

func (b *InMemoryBackend) DescribePersistentAppUI(ctx context.Context, id string) (*PersistentAppUI, error)

DescribePersistentAppUI returns a persistent app UI by ID.

func (*InMemoryBackend) DescribeReleaseLabel 

func (b *InMemoryBackend) DescribeReleaseLabel(
	_ context.Context, releaseLabel string,
) (*ReleaseLabel, error)

DescribeReleaseLabel returns details about a specific release label.

func (*InMemoryBackend) DescribeSecurityConfiguration 

func (b *InMemoryBackend) DescribeSecurityConfiguration(
	ctx context.Context,
	name string,
) (*SecurityConfiguration, error)

DescribeSecurityConfiguration returns the details of a security configuration.

func (*InMemoryBackend) DescribeStep 

func (b *InMemoryBackend) DescribeStep(ctx context.Context, clusterID, stepID string) (*Step, error)

DescribeStep returns a single step by cluster ID and step ID.

func (*InMemoryBackend) DescribeStudio 

func (b *InMemoryBackend) DescribeStudio(ctx context.Context, studioID string) (*Studio, error)

DescribeStudio returns an EMR Studio by ID.

func (*InMemoryBackend) GetAutoTerminationPolicy 

func (b *InMemoryBackend) GetAutoTerminationPolicy(
	ctx context.Context,
	clusterID string,
) (*AutoTerminationPolicy, error)

GetAutoTerminationPolicy returns the auto-termination policy for a cluster, or nil if none is attached -- see GetManagedScalingPolicy for why nil (not a zero-valued policy) must be propagated.

func (*InMemoryBackend) GetBlockPublicAccessConfiguration 

func (b *InMemoryBackend) GetBlockPublicAccessConfiguration(
	ctx context.Context,
) (BlockPublicAccessConfiguration, blockPublicAccessMeta)

GetBlockPublicAccessConfiguration returns the account-level block-public-access config.

func (*InMemoryBackend) GetClusterSessionCredentials 

func (b *InMemoryBackend) GetClusterSessionCredentials(
	ctx context.Context,
	clusterID, executionRoleArn string,
) (map[string]any, time.Time, error)

GetClusterSessionCredentials returns synthesized credentials for cluster session access.

func (*InMemoryBackend) GetManagedScalingPolicy 

func (b *InMemoryBackend) GetManagedScalingPolicy(
	ctx context.Context, clusterID string,
) (*ManagedScalingPolicy, error)

GetManagedScalingPolicy returns the managed scaling policy for a cluster, or nil if none is attached -- real GetManagedScalingPolicyOutput.ManagedScalingPolicy is a pointer that is omitted from the wire entirely when unset (not a zero-valued object), so callers must propagate nil rather than substitute an empty policy.

func (*InMemoryBackend) GetOnClusterPresignedURL 

func (b *InMemoryBackend) GetOnClusterPresignedURL(_ context.Context, clusterID, region string) (string, error)

GetOnClusterPresignedURL returns a presigned URL for an on-cluster app UI, verifying cluster exists.

func (*InMemoryBackend) GetPresignedURL 

func (b *InMemoryBackend) GetPresignedURL(id, region string) string

GetPresignedURL returns a synthetic presigned URL for a persistent app UI.

func (*InMemoryBackend) GetSession added in v1.2.0 

func (b *InMemoryBackend) GetSession(ctx context.Context, clusterID, sessionID string) (*Session, error)

GetSession returns a single session by cluster ID and session ID.

func (*InMemoryBackend) GetSessionEndpoint added in v1.2.0 

func (b *InMemoryBackend) GetSessionEndpoint(
	ctx context.Context,
	clusterID, sessionID string,
) (*SessionEndpointResult, error)

GetSessionEndpoint returns the Spark Connect endpoint URL, a synthesized auth token, and VPC-peering credentials for a session, verifying the cluster and session both exist.

func (*InMemoryBackend) GetStudioSessionMapping 

func (b *InMemoryBackend) GetStudioSessionMapping(
	ctx context.Context,
	studioID, identityType, identityID, identityName string,
) (*StudioSessionMapping, error)

GetStudioSessionMapping returns a session mapping for a studio.

func (*InMemoryBackend) ListBootstrapActions 

func (b *InMemoryBackend) ListBootstrapActions(
	ctx context.Context,
	clusterID, marker string,
) ([]Command, string, error)

ListBootstrapActions returns the bootstrap actions for a cluster, paginated.

func (*InMemoryBackend) ListClusters 

func (b *InMemoryBackend) ListClusters(ctx context.Context, params ListClustersParams) ([]ClusterSummary, string)

ListClusters returns cluster summaries matching the given filter, sorted by creation time descending.

func (*InMemoryBackend) ListInstanceFleets 

func (b *InMemoryBackend) ListInstanceFleets(ctx context.Context, clusterID string) ([]InstanceFleet, error)

ListInstanceFleets returns the instance fleets for a cluster by its ID.

func (*InMemoryBackend) ListInstanceGroups 

func (b *InMemoryBackend) ListInstanceGroups(ctx context.Context, clusterID string) ([]InstanceGroup, error)

ListInstanceGroups returns the instance groups for a cluster by its ID.

func (*InMemoryBackend) ListInstances 

func (b *InMemoryBackend) ListInstances(
	ctx context.Context,
	clusterID string,
	params ListInstancesParams,
) ([]ClusterInstance, string)

ListInstances synthesizes per-group instances for a cluster.

func (*InMemoryBackend) ListNotebookExecutions 

func (b *InMemoryBackend) ListNotebookExecutions(
	ctx context.Context, params ListNotebookExecutionsParams,
) ([]NotebookExecution, string)

ListNotebookExecutions returns paginated notebook executions matching the filter.

func (*InMemoryBackend) ListReleaseLabels 

func (b *InMemoryBackend) ListReleaseLabels(
	_ context.Context, prefix, application, marker string,
) ([]string, string)

ListReleaseLabels returns release labels optionally filtered by prefix and application.

func (*InMemoryBackend) ListSecurityConfigurations 

func (b *InMemoryBackend) ListSecurityConfigurations(
	ctx context.Context,
	marker string,
) ([]SecurityConfigSummary, string)

ListSecurityConfigurations returns all security configurations, sorted by name.

func (*InMemoryBackend) ListSessions added in v1.2.0 

func (b *InMemoryBackend) ListSessions(
	ctx context.Context,
	clusterID string,
	states []string,
	marker string,
) ([]Session, string, error)

ListSessions returns the sessions on a cluster, optionally filtered by state, newest first (matching real ListSessions' doc: "Newer sessions are returned first").

func (*InMemoryBackend) ListSteps 

func (b *InMemoryBackend) ListSteps(
	ctx context.Context,
	clusterID string,
	stepStates []string,
	stepIDs []string,
	marker string,
) ([]Step, string)

ListSteps returns steps for a cluster, optionally filtered by state and/or ID.

func (*InMemoryBackend) ListStudioSessionMappings 

func (b *InMemoryBackend) ListStudioSessionMappings(
	ctx context.Context,
	studioID, identityType string,
) []StudioSessionMapping

ListStudioSessionMappings returns session mappings for a studio, optionally filtered by identity type.

func (*InMemoryBackend) ListStudios 

func (b *InMemoryBackend) ListStudios(ctx context.Context, marker string) ([]StudioSummary, string)

ListStudios returns all studios as summaries, sorted by name.

func (*InMemoryBackend) ListSupportedInstanceTypes 

func (b *InMemoryBackend) ListSupportedInstanceTypes(
	_ context.Context, releaseLabel, marker string,
) ([]SupportedInstanceType, string)

ListSupportedInstanceTypes returns the static catalog of EMR-supported instance types.

func (*InMemoryBackend) ListTagsForResource 

func (b *InMemoryBackend) ListTagsForResource(ctx context.Context, resourceID string) ([]Tag, error)

ListTagsForResource returns tags for a cluster or studio identified by ARN or ID, sorted by key.

func (*InMemoryBackend) ModifyCluster 

func (b *InMemoryBackend) ModifyCluster(
	ctx context.Context, clusterID string, stepConcurrencyLevel int,
) (int, error)

ModifyCluster updates StepConcurrencyLevel on a cluster.

func (*InMemoryBackend) ModifyInstanceFleet 

func (b *InMemoryBackend) ModifyInstanceFleet(
	ctx context.Context,
	clusterID string,
	mod InstanceFleetModification,
) error

ModifyInstanceFleet updates target capacities on an instance fleet.

func (*InMemoryBackend) ModifyInstanceGroups 

func (b *InMemoryBackend) ModifyInstanceGroups(
	ctx context.Context,
	clusterID string,
	mods []InstanceGroupModification,
) error

ModifyInstanceGroups updates instance counts for the specified groups.

func (*InMemoryBackend) PutAutoScalingPolicy 

func (b *InMemoryBackend) PutAutoScalingPolicy(
	ctx context.Context,
	clusterID, instanceGroupID string,
	policy AutoScalingPolicySpec,
) (*AutoScalingPolicyDetail, string, string, error)

PutAutoScalingPolicy persists an auto-scaling policy on an instance group.

func (*InMemoryBackend) PutAutoTerminationPolicy 

func (b *InMemoryBackend) PutAutoTerminationPolicy(
	ctx context.Context,
	clusterID string,
	policy AutoTerminationPolicy,
) error

PutAutoTerminationPolicy sets the auto-termination policy on a cluster.

func (*InMemoryBackend) PutBlockPublicAccessConfiguration 

func (b *InMemoryBackend) PutBlockPublicAccessConfiguration(
	ctx context.Context,
	config BlockPublicAccessConfiguration,
) error

PutBlockPublicAccessConfiguration sets the account-level block-public-access config.

func (*InMemoryBackend) PutManagedScalingPolicy 

func (b *InMemoryBackend) PutManagedScalingPolicy(
	ctx context.Context,
	clusterID string,
	policy ManagedScalingPolicy,
) error

PutManagedScalingPolicy sets the managed scaling policy on a cluster.

func (*InMemoryBackend) Region 

func (b *InMemoryBackend) Region() string

Region returns the AWS region this backend is configured for.

func (*InMemoryBackend) RemoveAutoScalingPolicy 

func (b *InMemoryBackend) RemoveAutoScalingPolicy(
	ctx context.Context, clusterID, instanceGroupID string,
) error

RemoveAutoScalingPolicy clears the auto-scaling policy on an instance group.

func (*InMemoryBackend) RemoveAutoTerminationPolicy 

func (b *InMemoryBackend) RemoveAutoTerminationPolicy(ctx context.Context, clusterID string) error

RemoveAutoTerminationPolicy clears the auto-termination policy on a cluster.

func (*InMemoryBackend) RemoveManagedScalingPolicy 

func (b *InMemoryBackend) RemoveManagedScalingPolicy(ctx context.Context, clusterID string) error

RemoveManagedScalingPolicy clears the managed scaling policy on a cluster.

func (*InMemoryBackend) RemoveTags 

func (b *InMemoryBackend) RemoveTags(ctx context.Context, resourceID string, tagKeys []string) error

RemoveTags removes tags from a cluster or studio identified by ARN or ID.

func (*InMemoryBackend) Reset 

func (b *InMemoryBackend) Reset()

Reset clears all in-memory state from the backend.

func (*InMemoryBackend) Restore 

func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error

Restore loads backend state from a JSON snapshot. It implements persistence.Persistable.

func (*InMemoryBackend) RunJobFlow 

func (b *InMemoryBackend) RunJobFlow(ctx context.Context, params RunJobFlowParams) (*Cluster, error)

RunJobFlow creates a new EMR cluster.

func (*InMemoryBackend) SetKeepJobFlowAliveWhenNoSteps 

func (b *InMemoryBackend) SetKeepJobFlowAliveWhenNoSteps(
	ctx context.Context, jobFlowIDs []string, keep bool,
) error

SetKeepJobFlowAliveWhenNoSteps sets the KeepJobFlowAliveWhenNoSteps flag.

func (*InMemoryBackend) SetTerminationProtection 

func (b *InMemoryBackend) SetTerminationProtection(
	ctx context.Context, jobFlowIDs []string, protect bool,
) error

SetTerminationProtection sets the TerminationProtected flag on clusters.

func (*InMemoryBackend) SetUnhealthyNodeReplacement 

func (b *InMemoryBackend) SetUnhealthyNodeReplacement(
	ctx context.Context, jobFlowIDs []string, replace bool,
) error

SetUnhealthyNodeReplacement sets the UnhealthyNodeReplacement flag.

func (*InMemoryBackend) SetVisibleToAllUsers 

func (b *InMemoryBackend) SetVisibleToAllUsers(
	ctx context.Context, jobFlowIDs []string, visible bool,
) error

SetVisibleToAllUsers sets the VisibleToAllUsers flag.

func (*InMemoryBackend) Snapshot 

func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte

Snapshot serialises the backend state to JSON. It implements persistence.Persistable.

func (*InMemoryBackend) StartNotebookExecution 

func (b *InMemoryBackend) StartNotebookExecution(
	ctx context.Context,
	editorID, name, params, engineID string,
	tags []Tag,
) (*NotebookExecution, error)

StartNotebookExecution creates a new notebook execution in RUNNING state.

func (*InMemoryBackend) StartSession added in v1.2.0 

func (b *InMemoryBackend) StartSession(ctx context.Context, params StartSessionParams) (*Session, error)

StartSession creates and starts a new interactive session on a cluster. The referenced cluster must exist and be in a state that can host a session (see sessionCanStart).

func (*InMemoryBackend) StopNotebookExecution 

func (b *InMemoryBackend) StopNotebookExecution(ctx context.Context, id string) error

StopNotebookExecution transitions a RUNNING execution to STOPPED.

func (*InMemoryBackend) TaggedResources added in v1.2.0 

func (b *InMemoryBackend) TaggedResources() []TaggedEntry

TaggedResources returns every EMR cluster and studio ARN, across all regions, that currently has at least one tag. b.clusters and b.studios are each a single store.Table keyed by "region|id" (see regionKey), so All() on each already spans every region without a per-region loop.

func (*InMemoryBackend) TerminateJobFlows 

func (b *InMemoryBackend) TerminateJobFlows(ctx context.Context, ids []string) error

TerminateJobFlows marks the specified clusters as TERMINATED. Returns ValidationException if any cluster has termination protection.

func (*InMemoryBackend) TerminateSession added in v1.2.0 

func (b *InMemoryBackend) TerminateSession(ctx context.Context, clusterID, sessionID string) (*Session, error)

TerminateSession terminates an active session on a cluster.

func (*InMemoryBackend) UpdateStudio 

func (b *InMemoryBackend) UpdateStudio(
	ctx context.Context,
	studioID, name, description, defaultS3Location string,
	subnetIDs []string,
) error

UpdateStudio updates mutable fields on an EMR Studio. Real UpdateStudioInput.SubnetIds doc: "The list can include new subnet IDs, but must also include all of the subnet IDs previously associated with the Studio" -- i.e. it is a full replace, not a merge, so an empty/absent subnetIDs leaves the existing set untouched (same "empty string means unset" convention already used for name/description/defaultS3Location) while a non-empty one replaces it wholesale.

func (*InMemoryBackend) UpdateStudioSessionMapping 

func (b *InMemoryBackend) UpdateStudioSessionMapping(
	ctx context.Context,
	studioID, identityType, identityID, identityName, sessionPolicyArn string,
) error

UpdateStudioSessionMapping changes the SessionPolicyArn on a mapping.

type InstanceFleet 

type InstanceFleet struct {
	Status                      InstanceFleetStatus `json:"Status"`
	ID                          string              `json:"Id"`
	Name                        string              `json:"Name"`
	InstanceFleetType           string              `json:"InstanceFleetType"`
	TargetOnDemandCapacity      int                 `json:"TargetOnDemandCapacity"`
	TargetSpotCapacity          int                 `json:"TargetSpotCapacity"`
	ProvisionedOnDemandCapacity int                 `json:"ProvisionedOnDemandCapacity"`
	ProvisionedSpotCapacity     int                 `json:"ProvisionedSpotCapacity"`
}

InstanceFleet represents an EMR instance fleet returned by AddInstanceFleet.

type InstanceFleetModification 

type InstanceFleetModification struct {
	InstanceFleetID        string `json:"InstanceFleetId"`
	TargetOnDemandCapacity int    `json:"TargetOnDemandCapacity,omitempty"`
	TargetSpotCapacity     int    `json:"TargetSpotCapacity,omitempty"`
}

InstanceFleetModification describes a fleet target capacity change.

type InstanceFleetSpec 

type InstanceFleetSpec struct {
	Name                   string          `json:"Name"`
	InstanceFleetType      string          `json:"InstanceFleetType"`
	Configurations         []Configuration `json:"Configurations,omitempty"`
	TargetOnDemandCapacity int             `json:"TargetOnDemandCapacity"`
	TargetSpotCapacity     int             `json:"TargetSpotCapacity"`
}

InstanceFleetSpec is the input specification for an instance fleet.

type InstanceFleetStatus 

type InstanceFleetStatus struct {
	State string `json:"State"`
}

InstanceFleetStatus tracks the provisioning state of an EMR instance fleet.

type InstanceGroup 

type InstanceGroup struct {
	AutoScalingPolicy      *AutoScalingPolicyDetail `json:"AutoScalingPolicy,omitempty"`
	Status                 InstanceGroupStatus      `json:"Status"`
	ID                     string                   `json:"Id"`
	Name                   string                   `json:"Name"`
	Market                 string                   `json:"Market"`
	BidPrice               string                   `json:"BidPrice,omitempty"`
	InstanceGroupType      string                   `json:"InstanceGroupType"`
	InstanceType           string                   `json:"InstanceType"`
	Configurations         []Configuration          `json:"Configurations,omitempty"`
	RequestedInstanceCount int                      `json:"RequestedInstanceCount"`
	RunningInstanceCount   int                      `json:"RunningInstanceCount"`
}

InstanceGroup represents an EMR instance group returned by ListInstanceGroups.

type InstanceGroupModification 

type InstanceGroupModification struct {
	InstanceGroupID string `json:"InstanceGroupId"`
	InstanceCount   int    `json:"InstanceCount"`
}

InstanceGroupModification describes a single instance group count change.

type InstanceGroupSpec 

type InstanceGroupSpec struct {
	Name           string          `json:"Name"`
	Market         string          `json:"Market"`
	InstanceRole   string          `json:"InstanceRole"`
	InstanceType   string          `json:"InstanceType"`
	BidPrice       string          `json:"BidPrice,omitempty"`
	Configurations []Configuration `json:"Configurations,omitempty"`
	InstanceCount  int             `json:"InstanceCount"`
}

InstanceGroupSpec is the input specification for an instance group from RunJobFlow.

type InstanceGroupStatus 

type InstanceGroupStatus struct {
	State string `json:"State"`
}

InstanceGroupStatus is the status of an EMR instance group.

type Janitor 

type Janitor struct {
	Backend       *InMemoryBackend
	Interval      time.Duration
	TerminatedTTL time.Duration
	TaskTimeout   time.Duration
}

Janitor is the EMR background worker that sweeps terminated clusters after a configurable TTL, matching the AWS behavior where terminated clusters remain visible for approximately one hour.

func NewJanitor 

func NewJanitor(backend *InMemoryBackend, interval, terminatedTTL time.Duration) *Janitor

NewJanitor creates a new EMR Janitor for the given backend. If interval or terminatedTTL are zero, defaults are used.

func (*Janitor) Run 

func (j *Janitor) Run(ctx context.Context)

Run runs the janitor loop until ctx is cancelled.

func (*Janitor) SweepOnce 

func (j *Janitor) SweepOnce(ctx context.Context)

SweepOnce runs a single sweep pass. Exposed for testing.

type JobFlow 

type JobFlow struct {
	JobFlowID             string                       `json:"JobFlowId"`
	Name                  string                       `json:"Name"`
	ReleaseLabel          string                       `json:"ReleaseLabel,omitempty"`
	LogURI                string                       `json:"LogUri,omitempty"`
	ServiceRole           string                       `json:"ServiceRole,omitempty"`
	Instances             JobFlowInstancesDetail       `json:"Instances"`
	ExecutionStatusDetail JobFlowExecutionStatusDetail `json:"ExecutionStatusDetail"`
}

JobFlow is the legacy format returned by DescribeJobFlows.

type JobFlowExecutionStatusDetail 

type JobFlowExecutionStatusDetail struct {
	State             string  `json:"State"`
	StateChangeReason string  `json:"StateChangeReason,omitempty"`
	CreationDateTime  float64 `json:"CreationDateTime"`
	EndDateTime       float64 `json:"EndDateTime,omitempty"`
}

JobFlowExecutionStatusDetail holds the legacy execution status.

type JobFlowInstancesDetail 

type JobFlowInstancesDetail struct {
	MasterInstanceType string `json:"MasterInstanceType,omitempty"`
	SlaveInstanceType  string `json:"SlaveInstanceType,omitempty"`
	InstanceCount      int    `json:"InstanceCount"`
}

JobFlowInstancesDetail holds the legacy instances detail.

type KerberosAttributes added in v1.2.0 

type KerberosAttributes struct {
	Realm                            string `json:"Realm"`
	KdcAdminPassword                 string `json:"KdcAdminPassword"`
	ADDomainJoinUser                 string `json:"ADDomainJoinUser,omitempty"`
	ADDomainJoinPassword             string `json:"ADDomainJoinPassword,omitempty"`
	CrossRealmTrustPrincipalPassword string `json:"CrossRealmTrustPrincipalPassword,omitempty"`
}

KerberosAttributes holds Kerberos configuration for a cluster, set via RunJobFlow and echoed back on Cluster when Kerberos authentication is enabled using a security configuration.

type ListClustersParams 

type ListClustersParams struct {
	CreatedAfter  *time.Time
	CreatedBefore *time.Time
	Marker        string
	ClusterStates []string
}

ListClustersParams holds filter and pagination params for ListClusters.

type ListInstancesParams 

type ListInstancesParams struct {
	InstanceGroupID    string
	InstanceFleetID    string
	Marker             string
	InstanceGroupTypes []string
	InstanceStates     []string
}

ListInstancesParams holds filter params for ListInstances.

type ListNotebookExecutionsParams 

type ListNotebookExecutionsParams struct {
	EditorID string
	Status   string
	Marker   string
}

ListNotebookExecutionsParams holds filters for ListNotebookExecutions.

type ManagedScalingPolicy 

type ManagedScalingPolicy struct {
	ComputeLimits ComputeLimits `json:"ComputeLimits"`
}

ManagedScalingPolicy defines managed scaling for a cluster.

type NotebookExecution 

type NotebookExecution struct {
	NotebookExecutionID   string `json:"NotebookExecutionId"`
	EditorID              string `json:"EditorId,omitempty"`
	NotebookExecutionName string `json:"NotebookExecutionName,omitempty"`
	NotebookParams        string `json:"NotebookParams,omitempty"`
	ExecutionEngineID     string `json:"ExecutionEngineId,omitempty"`
	Status                string `json:"Status"`

	Tags      []Tag   `json:"Tags"`
	StartTime float64 `json:"StartTime,omitempty"`
	EndTime   float64 `json:"EndTime,omitempty"`
	// contains filtered or unexported fields
}

NotebookExecution represents an EMR Studio notebook execution.

StartTime/EndTime are epoch seconds (float64), matching the EMR awsjson1.1 wire format -- the real SDK deserializer parses these with smithytime.ParseEpochSeconds and rejects RFC3339 strings. A zero value (unset) is omitted via omitempty, matching the "not yet ended" case where AWS omits EndTime entirely.

type PersistentAppUI 

type PersistentAppUI struct {
	ID                string `json:"PersistentAppUIId"`
	TargetResourceArn string `json:"TargetResourceArn"`

	RuntimeRoleEnabledCluster bool `json:"RuntimeRoleEnabledCluster"`
	// contains filtered or unexported fields
}

PersistentAppUI represents an EMR persistent application user interface.

type PlacementGroupConfig added in v1.2.0 

type PlacementGroupConfig struct {
	InstanceRole      string `json:"InstanceRole"`
	PlacementStrategy string `json:"PlacementStrategy,omitempty"`
}

PlacementGroupConfig is the placement group configuration for a single instance role, part of RunJobFlow's Instances.Placement and echoed back on Cluster.PlacementGroups.

type PortRange 

type PortRange struct {
	MinRange int `json:"MinRange"`
	MaxRange int `json:"MaxRange"`
}

PortRange defines an inclusive range of port numbers.

type Provider 

type Provider struct{}

Provider implements service.Provider for EMR.

func (*Provider) Init 

func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error)

Init initializes the EMR backend and handler.

func (*Provider) Name 

func (p *Provider) Name() string

Name returns the provider name.

type ReleaseLabel 

type ReleaseLabel struct {
	ReleaseLabel string                    `json:"ReleaseLabel"`
	Applications []ReleaseLabelApplication `json:"Applications,omitempty"`
}

ReleaseLabel holds details about an EMR release label.

type ReleaseLabelApplication 

type ReleaseLabelApplication struct {
	Name    string `json:"Name"`
	Version string `json:"Version"`
}

ReleaseLabelApplication is an application listed for a release label.

type RunJobFlowInstances 

type RunJobFlowInstances struct {
	Ec2KeyName                     string              `json:"Ec2KeyName,omitempty"`
	Ec2SubnetID                    string              `json:"Ec2SubnetId,omitempty"`
	EmrManagedMasterSecurityGroup  string              `json:"EmrManagedMasterSecurityGroup,omitempty"`
	EmrManagedSlaveSecurityGroup   string              `json:"EmrManagedSlaveSecurityGroup,omitempty"`
	ServiceAccessSecurityGroup     string              `json:"ServiceAccessSecurityGroup,omitempty"`
	InstanceGroups                 []InstanceGroupSpec `json:"InstanceGroups,omitempty"`
	InstanceFleets                 []InstanceFleetSpec `json:"InstanceFleets,omitempty"`
	Ec2SubnetIDs                   []string            `json:"Ec2SubnetIds,omitempty"`
	AdditionalMasterSecurityGroups []string            `json:"AdditionalMasterSecurityGroups,omitempty"`
	AdditionalSlaveSecurityGroups  []string            `json:"AdditionalSlaveSecurityGroups,omitempty"`
	KeepJobFlowAliveWhenNoSteps    bool                `json:"KeepJobFlowAliveWhenNoSteps"`
	TerminationProtected           bool                `json:"TerminationProtected"`
}

RunJobFlowInstances holds the Instances block from a RunJobFlow call.

NOTE: real EMR's JobFlowInstancesConfig has no IamInstanceProfile member -- that attribute is set via the top-level RunJobFlowInput.JobFlowRole field instead (see RunJobFlowParams.JobFlowRole) and echoed back on Cluster.Ec2InstanceAttributes.IamInstanceProfile. An IamInstanceProfile field used to live here, but no real client ever populates it at this nesting level, so it was deleted.

type RunJobFlowParams 

type RunJobFlowParams struct {
	SecurityConfiguration string `json:"SecurityConfiguration,omitempty"`
	ReleaseLabel          string `json:"ReleaseLabel"`
	OSReleaseLabel        string `json:"OSReleaseLabel,omitempty"`
	LogURI                string `json:"LogUri,omitempty"`
	ServiceRole           string `json:"ServiceRole,omitempty"`
	// JobFlowRole is the real RunJobFlowInput field (also called the EC2
	// instance profile); it becomes Cluster.Ec2InstanceAttributes.IamInstanceProfile.
	JobFlowRole             string                  `json:"JobFlowRole,omitempty"`
	AutoScalingRole         string                  `json:"AutoScalingRole,omitempty"`
	Name                    string                  `json:"Name"`
	ScaleDownBehavior       string                  `json:"ScaleDownBehavior,omitempty"`
	CustomAmiID             string                  `json:"CustomAmiId,omitempty"`
	Tags                    []Tag                   `json:"Tags,omitempty"`
	Applications            []Application           `json:"Applications,omitempty"`
	Configurations          []Configuration         `json:"Configurations,omitempty"`
	Steps                   []StepSpec              `json:"Steps,omitempty"`
	BootstrapActions        []BootstrapActionConfig `json:"BootstrapActions,omitempty"`
	KerberosAttributes      *KerberosAttributes     `json:"KerberosAttributes,omitempty"`
	PlacementGroupConfigs   []PlacementGroupConfig  `json:"PlacementGroupConfigs,omitempty"`
	ManagedScalingPolicy    *ManagedScalingPolicy   `json:"ManagedScalingPolicy,omitempty"`
	AutoTerminationPolicy   *AutoTerminationPolicy  `json:"AutoTerminationPolicy,omitempty"`
	Instances               RunJobFlowInstances     `json:"Instances"`
	StepConcurrencyLevel    int                     `json:"StepConcurrencyLevel,omitempty"`
	EbsRootVolumeSize       int                     `json:"EbsRootVolumeSize,omitempty"`
	EbsRootVolumeIops       int                     `json:"EbsRootVolumeIops,omitempty"`
	EbsRootVolumeThroughput int                     `json:"EbsRootVolumeThroughput,omitempty"`
	VisibleToAllUsers       bool                    `json:"VisibleToAllUsers"`
}

RunJobFlowParams is the full input for creating a new cluster.

type ScalingAction 

type ScalingAction struct {
	SimpleScalingPolicyConfiguration SimpleScalingPolicyConfiguration `json:"SimpleScalingPolicyConfiguration"`
}

ScalingAction defines what to do when a scaling rule fires.

type ScalingRule 

type ScalingRule struct {
	Name        string         `json:"Name"`
	Description string         `json:"Description,omitempty"`
	Action      ScalingAction  `json:"Action"`
	Trigger     ScalingTrigger `json:"Trigger"`
}

ScalingRule is a named auto-scaling rule combining action and trigger.

type ScalingTrigger 

type ScalingTrigger struct {
	CloudWatchAlarmDefinition CloudWatchAlarmDefinition `json:"CloudWatchAlarmDefinition"`
}

ScalingTrigger wraps a CloudWatch alarm definition.

type SecurityConfigSummary 

type SecurityConfigSummary struct {
	Name             string  `json:"Name"`
	CreationDateTime float64 `json:"CreationDateTime"`
}

SecurityConfigSummary is returned by ListSecurityConfigurations. CreationDateTime is epoch seconds (float64); see SecurityConfiguration for why.

type SecurityConfiguration 

type SecurityConfiguration struct {
	Name           string `json:"Name"`
	SecurityConfig string `json:"SecurityConfiguration"`

	CreationDateTime float64 `json:"CreationDateTime"`
	// contains filtered or unexported fields
}

SecurityConfiguration stores an EMR security configuration.

CreationDateTime is epoch seconds (float64), matching the EMR awsjson1.1 wire format -- the real SDK deserializer parses CreationDateTime fields with smithytime.ParseEpochSeconds and rejects RFC3339 strings.

type Session added in v1.2.0 

type Session struct {
	MonitoringConfiguration     *SessionMonitoringConfiguration `json:"MonitoringConfiguration,omitempty"`
	ID                          string                          `json:"Id"`
	ClusterID                   string                          `json:"ClusterId"`
	ARN                         string                          `json:"Arn"`
	State                       string                          `json:"State"`
	AccountID                   string                          `json:"AccountId,omitempty"`
	Name                        string                          `json:"Name,omitempty"`
	ExecutionRoleArn            string                          `json:"ExecutionRoleArn,omitempty"`
	ReleaseLabel                string                          `json:"ReleaseLabel,omitempty"`
	ServerURL                   string                          `json:"ServerUrl,omitempty"`
	StateChangeReason           string                          `json:"StateChangeReason,omitempty"`
	EngineConfigurations        []Configuration                 `json:"EngineConfigurations,omitempty"`
	Tags                        []Tag                           `json:"Tags"`
	SessionIdleTimeoutInMinutes int64                           `json:"SessionIdleTimeoutInMinutes,omitempty"`
	CreatedAt                   float64                         `json:"CreatedAt"`
	UpdatedAt                   float64                         `json:"UpdatedAt"`
	EndedAt                     float64                         `json:"EndedAt,omitempty"`
	IdleSince                   float64                         `json:"IdleSince,omitempty"`
	StartedAt                   float64                         `json:"StartedAt,omitempty"`
}

Session represents an interactive (Spark Connect) session running on an EMR cluster (types.Session). See sessions.go's package doc comment for the state-model rationale.

CreatedAt/UpdatedAt are always populated once a session exists, so they carry no omitempty (matching Cluster/SecurityConfiguration's convention for "always set" epoch-seconds fields elsewhere in this package). EndedAt/IdleSince/StartedAt are genuine real Timestamp members this backend only sometimes populates (IdleSince/StartedAt are never populated at all -- see sessions.go), so they use omitempty, matching the real optional *time.Time members on types.Session. All are epoch seconds (float64), matching EMR's awsjson1.1 wire format -- see SecurityConfiguration for why.

type SessionCloudWatchLoggingConfiguration added in v1.2.0 

type SessionCloudWatchLoggingConfiguration struct {
	LogTypes            map[string][]string `json:"LogTypes,omitempty"`
	EncryptionKeyArn    string              `json:"EncryptionKeyArn,omitempty"`
	LogGroup            string              `json:"LogGroup,omitempty"`
	LogStreamNamePrefix string              `json:"LogStreamNamePrefix,omitempty"`
	Enabled             bool                `json:"Enabled,omitempty"`
}

SessionCloudWatchLoggingConfiguration is the CloudWatch Logs configuration for a session (types.SessionCloudWatchLoggingConfiguration).

type SessionEndpointResult added in v1.2.0 

type SessionEndpointResult struct {
	Expiry      time.Time
	Credentials map[string]any
	Endpoint    string
	AuthToken   string
}

SessionEndpointResult is the output of GetSessionEndpoint.

type SessionManagedLoggingConfiguration added in v1.2.0 

type SessionManagedLoggingConfiguration struct {
	EncryptionKeyArn string `json:"EncryptionKeyArn,omitempty"`
	Enabled          bool   `json:"Enabled,omitempty"`
}

SessionManagedLoggingConfiguration is the Amazon EMR-managed logging configuration for a session (types.SessionManagedLoggingConfiguration).

type SessionMonitoringConfiguration added in v1.2.0 

type SessionMonitoringConfiguration struct {
	CloudWatchLoggingConfiguration *SessionCloudWatchLoggingConfiguration `json:"CloudWatchLoggingConfiguration,omitempty"`
	ManagedLoggingConfiguration    *SessionManagedLoggingConfiguration    `json:"ManagedLoggingConfiguration,omitempty"`
	S3LoggingConfiguration         *SessionS3LoggingConfiguration         `json:"S3LoggingConfiguration,omitempty"`
}

SessionMonitoringConfiguration controls where a session's logs are published (types.SessionMonitoringConfiguration).

type SessionS3LoggingConfiguration added in v1.2.0 

type SessionS3LoggingConfiguration struct {
	LogTypes         map[string][]string `json:"LogTypes,omitempty"`
	EncryptionKeyArn string              `json:"EncryptionKeyArn,omitempty"`
	LogURI           string              `json:"LogUri,omitempty"`
	Enabled          bool                `json:"Enabled,omitempty"`
}

SessionS3LoggingConfiguration is the Amazon S3 logging configuration for a session (types.SessionS3LoggingConfiguration).

type Settings 

type Settings struct {
	JanitorInterval time.Duration `json:"janitor_interval" env:"EMR_JANITOR_INTERVAL" default:"1m" help:"Janitor tick interval."`                               //nolint:lll // Kong struct tag makes this line long
	TerminatedTTL   time.Duration `json:"terminated_ttl"   env:"EMR_TERMINATED_TTL"   default:"1h" help:"TTL for terminated clusters before they are evicted."` //nolint:lll // Kong struct tag makes this line long
}

Settings holds service-level configuration for the EMR backend. Fields are picked up by the Kong CLI parser when this struct is embedded in the root CLI command.

type SimpleScalingPolicyConfiguration 

type SimpleScalingPolicyConfiguration struct {
	AdjustmentType    string `json:"AdjustmentType"`
	ScalingAdjustment int    `json:"ScalingAdjustment"`
	CoolDown          int    `json:"CoolDown,omitempty"`
}

SimpleScalingPolicyConfiguration defines scaling adjustment details.

type StartSessionParams added in v1.2.0 

type StartSessionParams struct {
	MonitoringConfiguration     *SessionMonitoringConfiguration
	ClusterID                   string
	Name                        string
	ExecutionRoleArn            string
	EngineConfigurations        []Configuration
	Tags                        []Tag
	SessionIdleTimeoutInMinutes int64
}

StartSessionParams is the input for creating a new session.

type Step 

type Step struct {
	ID              string            `json:"Id"`
	Name            string            `json:"Name"`
	HadoopJarStep   StepHadoopJarStep `json:"HadoopJarStep"`
	ActionOnFailure string            `json:"ActionOnFailure"`
	Status          StepStatus        `json:"Status"`
}

Step represents an EMR step attached to a cluster.

type StepHadoopJarStep 

type StepHadoopJarStep struct {
	Jar       string   `json:"Jar"`
	MainClass string   `json:"MainClass,omitempty"`
	Args      []string `json:"Args,omitempty"`
}

StepHadoopJarStep defines the JAR execution for a step.

type StepSpec 

type StepSpec struct {
	Name            string            `json:"Name"`
	ActionOnFailure string            `json:"ActionOnFailure"`
	HadoopJarStep   StepHadoopJarStep `json:"HadoopJarStep"`
}

StepSpec is the input for adding a new step.

type StepStatus 

type StepStatus struct {
	State    string       `json:"State"`
	Timeline StepTimeline `json:"Timeline"`
}

StepStatus holds the lifecycle state of an EMR step.

type StepTimeline 

type StepTimeline struct {
	CreationDateTime float64 `json:"CreationDateTime"`
	StartDateTime    float64 `json:"StartDateTime,omitempty"`
	EndDateTime      float64 `json:"EndDateTime,omitempty"`
}

StepTimeline tracks creation and completion times of a step.

type Studio 

type Studio struct {
	EngineSecurityGroupID      string `json:"EngineSecurityGroupId"`
	VpcID                      string `json:"VpcId"`
	StudioID                   string `json:"StudioId"`
	EncryptionKeyArn           string `json:"EncryptionKeyArn,omitempty"`
	Name                       string `json:"Name"`
	Description                string `json:"Description,omitempty"`
	AuthMode                   string `json:"AuthMode"`
	DefaultS3Location          string `json:"DefaultS3Location"`
	ServiceRole                string `json:"ServiceRole"`
	IdcInstanceArn             string `json:"IdcInstanceArn,omitempty"`
	URL                        string `json:"Url"`
	WorkspaceSecurityGroupID   string `json:"WorkspaceSecurityGroupId"`
	StudioArn                  string `json:"StudioArn"`
	UserRole                   string `json:"UserRole,omitempty"`
	IdpAuthURL                 string `json:"IdpAuthUrl,omitempty"`
	IdpRelayStateParameterName string `json:"IdpRelayStateParameterName,omitempty"`

	Tags                              []Tag    `json:"Tags"`
	SubnetIDs                         []string `json:"SubnetIds"`
	CreationTime                      float64  `json:"CreationTime,omitempty"`
	TrustedIdentityPropagationEnabled bool     `json:"TrustedIdentityPropagationEnabled"`
	// contains filtered or unexported fields
}

Studio represents an EMR Studio.

CreationTime is epoch seconds (float64), matching the EMR awsjson1.1 wire format -- the real SDK deserializer parses CreationTime with smithytime.ParseEpochSeconds and rejects RFC3339 strings.

type StudioSessionMapping 

type StudioSessionMapping struct {
	StudioID         string `json:"StudioId"`
	IdentityType     string `json:"IdentityType"`
	IdentityID       string `json:"IdentityId,omitempty"`
	IdentityName     string `json:"IdentityName,omitempty"`
	SessionPolicyArn string `json:"SessionPolicyArn"`

	LastModifiedTime float64 `json:"LastModifiedTime,omitempty"`
	CreationTime     float64 `json:"CreationTime,omitempty"`
	// contains filtered or unexported fields
}

StudioSessionMapping maps a user or group to an EMR Studio. CreationTime/LastModifiedTime are epoch seconds (float64); see Studio for why.

type StudioSummary 

type StudioSummary struct {
	StudioID          string  `json:"StudioId"`
	StudioArn         string  `json:"StudioArn"`
	Name              string  `json:"Name"`
	VpcID             string  `json:"VpcId"`
	DefaultS3Location string  `json:"DefaultS3Location"`
	AuthMode          string  `json:"AuthMode"`
	URL               string  `json:"Url"`
	Description       string  `json:"Description,omitempty"`
	CreationTime      float64 `json:"CreationTime,omitempty"`
}

StudioSummary is a trimmed view of Studio for ListStudios. CreationTime is epoch seconds (float64); see Studio for why.

type SupportedInstanceType 

type SupportedInstanceType struct {
	Type          string  `json:"Type"`
	Architecture  string  `json:"Architecture"`
	MemoryGB      float64 `json:"MemoryGB"`
	VCPU          int     `json:"VCPU"`
	NumberOfDisks int     `json:"NumberOfDisks,omitempty"`
	Is64BitsOnly  bool    `json:"Is64BitsOnly"`
}

SupportedInstanceType describes an EC2 instance type supported by EMR.

type Tag 

type Tag struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}

Tag is an EMR resource tag.

type TaggedEntry added in v1.2.0 

type TaggedEntry struct {
	Tags map[string]string
	ARN  string
}

TaggedEntry pairs a resource ARN with its tag map, for cross-service tag enumeration by the Resource Groups Tagging API (see cli.go's wireTaggingEMR).

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.