runner

package
v2.7.0-rc.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 28, 2026 License: MIT Imports: 21 Imported by: 0

Documentation

Overview

Package runner provides the test runners that run sets of test cases. Supported runners: rspec, jest, cypress, playwright, pytest, gotest, cucumber, nunit.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Cucumber 

type Cucumber struct {
	RunnerConfig
}

func NewCucumber 

func NewCucumber(c RunnerConfig) Cucumber

func (Cucumber) CommandNameAndArgs 

func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

CommandNameAndArgs replaces placeholders and returns command + args.

func (Cucumber) GetExamples 

func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error)

GetExamples returns an array of test scenarios within the given feature files.

func (Cucumber) GetFiles 

func (c Cucumber) GetFiles() ([]string, error)

GetFiles returns the list of feature files based on include / exclude pattern.

func (Cucumber) Name 

func (c Cucumber) Name() string

func (Cucumber) ParseReport 

func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)

func (Cucumber) ResultFormat added in v2.7.0 

func (c Cucumber) ResultFormat() string

func (Cucumber) Run 

func (c Cucumber) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes the Cucumber command and records results.

func (Cucumber) SupportedFeatures added in v2.7.0 

func (c Cucumber) SupportedFeatures() SupportedFeatures

type CucumberDataTableRow 

type CucumberDataTableRow struct {
	Cells []string `json:"cells"`
}

CucumberDataTableRow represents a row in a step's data table.

type CucumberDocString 

type CucumberDocString struct {
	Value       string `json:"value"`
	ContentType string `json:"content_type"`
	Line        int    `json:"line"`
}

CucumberDocString represents a doc string argument for a step.

type CucumberElement 

type CucumberElement struct {
	ID          string         `json:"id"`
	Keyword     string         `json:"keyword"`
	Name        string         `json:"name"`
	Description string         `json:"description"`
	Line        int            `json:"line"`
	Type        string         `json:"type"` // e.g., "scenario", "background"
	Steps       []CucumberStep `json:"steps"`
	Tags        []CucumberTag  `json:"tags,omitempty"`
}

CucumberElement represents a scenario or background in Cucumber's JSON output.

func (CucumberElement) AggregatedStatus 

func (e CucumberElement) AggregatedStatus() string

AggregatedStatus returns overall scenario status based on its steps. It mirrors the logic previously in cucumber.go but uses the parser's structs.

type CucumberFeature 

type CucumberFeature struct {
	URI         string            `json:"uri"`
	ID          string            `json:"id"`
	Keyword     string            `json:"keyword"`
	Name        string            `json:"name"`
	Description string            `json:"description"`
	Line        int               `json:"line"`
	Elements    []CucumberElement `json:"elements"`
	Tags        []CucumberTag     `json:"tags,omitempty"`
}

CucumberFeature represents a single feature in Cucumber's JSON output.

func ParseCucumberJSONReport 

func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error)

ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). This is for actual test results.

type CucumberMatch 

type CucumberMatch struct {
	Location string `json:"location"` // e.g., "features/step_definitions/steps.rb:5"
}

CucumberMatch represents the matching step definition for a step.

type CucumberResult 

type CucumberResult struct {
	Status       string `json:"status"` // e.g., "passed", "failed", "skipped", "undefined"
	ErrorMessage string `json:"error_message,omitempty"`
	Duration     int64  `json:"duration,omitempty"` // Nanoseconds
}

CucumberResult represents the result of a step execution.

type CucumberStep 

type CucumberStep struct {
	Keyword       string                 `json:"keyword"`
	Name          string                 `json:"name"`
	Line          int                    `json:"line"`
	Result        *CucumberResult        `json:"result,omitempty"`
	Match         *CucumberMatch         `json:"match,omitempty"`
	DocString     *CucumberDocString     `json:"doc_string,omitempty"`
	DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables
}

CucumberStep represents a single step in a scenario.

type CucumberTag 

type CucumberTag struct {
	Name string `json:"name"`
	Line int    `json:"line"`
}

CucumberTag represents a tag in Cucumber.

type Custom 

type Custom struct {
	RunnerConfig
}

func NewCustom 

func NewCustom(r RunnerConfig) (Custom, error)

func (Custom) CommandNameAndArgs 

func (r Custom) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (Custom) GetExamples 

func (r Custom) GetExamples(files []string) ([]plan.TestCase, error)

func (Custom) GetFiles 

func (r Custom) GetFiles() ([]string, error)

GetFiles returns an array of file names using the discovery pattern.

func (Custom) Name 

func (r Custom) Name() string

func (Custom) Run 

func (r Custom) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

func (Custom) SupportedFeatures added in v2.7.0 

func (c Custom) SupportedFeatures() SupportedFeatures

type Cypress 

type Cypress struct {
	RunnerConfig
}

func NewCypress 

func NewCypress(c RunnerConfig) Cypress

func (Cypress) CommandNameAndArgs 

func (c Cypress) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (Cypress) GetExamples 

func (c Cypress) GetExamples(files []string) ([]plan.TestCase, error)

func (Cypress) GetFiles 

func (c Cypress) GetFiles() ([]string, error)

func (Cypress) Name 

func (c Cypress) Name() string

func (Cypress) Run 

func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

func (Cypress) SupportedFeatures added in v2.7.0 

func (c Cypress) SupportedFeatures() SupportedFeatures

type GoTest 

type GoTest struct {
	RunnerConfig
}

func NewGoTest 

func NewGoTest(c RunnerConfig) GoTest

func (GoTest) CommandNameAndArgs 

func (g GoTest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (GoTest) GetExamples 

func (g GoTest) GetExamples(files []string) ([]plan.TestCase, error)

func (GoTest) GetFiles 

func (g GoTest) GetFiles() ([]string, error)

GetFiles discovers Go packages using `go list ./...`. Note that "file" does not exist as a first level concept in Golang projects So this func is returning a list of packages instead of files. The implication is that the Server-side smart test splitting will never work. It almost will always fallback to simple splitting.

func (GoTest) Name 

func (g GoTest) Name() string

func (GoTest) ResultFormat added in v2.7.0 

func (g GoTest) ResultFormat() string

func (GoTest) Run 

func (g GoTest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes the configured command for the specified packages.

func (GoTest) SupportedFeatures added in v2.7.0 

func (g GoTest) SupportedFeatures() SupportedFeatures

type JUnitXMLError added in v2.7.0 

type JUnitXMLError struct {
	Message string `xml:"message,attr"`
	Type    string `xml:"type,attr"`
	Content string `xml:",chardata"`
}

JUnitXMLError represents the <error> element in JUnit XML

type JUnitXMLFailure 

type JUnitXMLFailure struct {
	Message string `xml:"message,attr"`
	Type    string `xml:"type,attr"`
	Content string `xml:",chardata"`
}

JUnitXMLFailure represents the <failure> element in JUnit XML

type JUnitXMLSkipped 

type JUnitXMLSkipped struct {
	Message string `xml:"message,attr"`
}

JUnitXMLSkipped represents the <skipped> element in JUnit XML

type JUnitXMLTestCase added in v2.7.0 

type JUnitXMLTestCase struct {
	Classname string           `xml:"classname,attr"`
	Name      string           `xml:"name,attr"`
	Result    TestStatus       // passed | failed | skipped
	Failure   *JUnitXMLFailure `xml:"failure"`
	Error     *JUnitXMLError   `xml:"error"`
	Skipped   *JUnitXMLSkipped `xml:"skipped"`
}

JUnitXMLTestCase represents a single <testcase> element in JUnit XML.

type Jest 

type Jest struct {
	RunnerConfig
}

func NewJest 

func NewJest(j RunnerConfig) Jest

func (Jest) CommandNameAndArgs 

func (j Jest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (Jest) GetExamples 

func (j Jest) GetExamples(files []string) ([]plan.TestCase, error)

func (Jest) GetFiles 

func (j Jest) GetFiles() ([]string, error)

GetFiles returns an array of file names using the discovery pattern.

func (Jest) Name 

func (j Jest) Name() string

func (Jest) ParseReport 

func (j Jest) ParseReport(path string) (JestReport, error)

func (Jest) ResultFormat added in v2.7.0 

func (j Jest) ResultFormat() string

func (Jest) Run 

func (j Jest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

func (Jest) SupportedFeatures added in v2.7.0 

func (j Jest) SupportedFeatures() SupportedFeatures

type JestExample 

type JestExample struct {
	Name           string   `json:"fullName"`
	Status         string   `json:"status"`
	Title          string   `json:"title"`
	AncestorTitles []string `json:"ancestorTitles"`
	Location       struct {
		Line   int
		Column int
	}
}

type JestReport 

type JestReport struct {
	NumFailedTests int
	TestResults    []struct {
		AssertionResults []JestExample
		FileName         string `json:"name"`
		Status           string
	}
}

type NUnit 

type NUnit struct {
	RunnerConfig
}

func NewNUnit 

func NewNUnit(c RunnerConfig) NUnit

func (NUnit) CommandNameAndArgs 

func (n NUnit) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (NUnit) GetExamples 

func (n NUnit) GetExamples(files []string) ([]plan.TestCase, error)

func (NUnit) GetFiles 

func (n NUnit) GetFiles() ([]string, error)

GetFiles returns an array of .cs test file names using the discovery pattern.

func (NUnit) Name 

func (n NUnit) Name() string

func (NUnit) ResultFormat added in v2.7.0 

func (n NUnit) ResultFormat() string

func (NUnit) Run 

func (n NUnit) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes dotnet test with a --filter expression built from the test cases. Test cases are mapped from .cs file paths to class names, and joined into a FullyQualifiedName~ filter expression.

func (NUnit) SupportedFeatures added in v2.7.0 

func (n NUnit) SupportedFeatures() SupportedFeatures

type Playwright 

type Playwright struct {
	RunnerConfig
}

func NewPlaywright 

func NewPlaywright(p RunnerConfig) Playwright

func (Playwright) CommandNameAndArgs 

func (p Playwright) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (Playwright) GetExamples 

func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)

GetExamples returns an array of test examples within the given files.

func (Playwright) GetFiles 

func (p Playwright) GetFiles() ([]string, error)

func (Playwright) Name 

func (p Playwright) Name() string

func (Playwright) ResultFormat added in v2.7.0 

func (p Playwright) ResultFormat() string

func (Playwright) Run 

func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

func (Playwright) SupportedFeatures added in v2.7.0 

func (p Playwright) SupportedFeatures() SupportedFeatures

type PlaywrightReport 

type PlaywrightReport struct {
	Suites []PlaywrightReportSuite
	Stats  struct {
		Expected   int
		Unexpected int
	}
	Errors []struct {
		Message string
	}
}

type PlaywrightReportSuite 

type PlaywrightReportSuite struct {
	Title  string
	Specs  []PlaywrightSpec
	Suites []PlaywrightReportSuite
}

type PlaywrightSpec 

type PlaywrightSpec struct {
	File   string
	Line   int
	Column int
	ID     string
	Title  string
	Ok     bool
	Tests  []PlaywrightTest
}

type PlaywrightTest 

type PlaywrightTest struct {
	ProjectName string
	Status      string
}

type ProcessSignaledError 

type ProcessSignaledError struct {
	Signal syscall.Signal
}

func (*ProcessSignaledError) Error 

func (e *ProcessSignaledError) Error() string

type Pytest 

type Pytest struct {
	RunnerConfig
	// contains filtered or unexported fields
}

func NewPytest 

func NewPytest(c RunnerConfig) Pytest

func (Pytest) CommandNameAndArgs 

func (p Pytest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (Pytest) GetExamples 

func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error)

GetExamples returns an array of test examples within the given files. It uses `pytest --collect-only -q` to enumerate individual tests.

--tag-filters can be used to filter tests by markers if specified. e.g. --tag-fitlers team:frontend matches markers: with @pytest.mark.execution_tag('team', 'frontend')

The --tag-filters feature also assumes Python Test Collector plugin version >1.2.0 is installed.

func (Pytest) GetFiles 

func (p Pytest) GetFiles() ([]string, error)

func (Pytest) Name 

func (p Pytest) Name() string

func (Pytest) ResultFormat added in v2.7.0 

func (p Pytest) ResultFormat() string

func (Pytest) Run 

func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

func (Pytest) SupportedFeatures added in v2.7.0 

func (p Pytest) SupportedFeatures() SupportedFeatures

type PytestPants 

type PytestPants struct {
	RunnerConfig
}

func NewPytestPants 

func NewPytestPants(c RunnerConfig) PytestPants

func (PytestPants) CommandNameAndArgs 

func (p PytestPants) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

func (PytestPants) GetExamples 

func (p PytestPants) GetExamples(files []string) ([]plan.TestCase, error)

func (PytestPants) GetFiles 

func (p PytestPants) GetFiles() ([]string, error)

func (PytestPants) Name 

func (p PytestPants) Name() string

func (PytestPants) Run 

func (p PytestPants) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

func (PytestPants) SupportedFeatures added in v2.7.0 

func (p PytestPants) SupportedFeatures() SupportedFeatures

type Rspec 

type Rspec struct {
	RunnerConfig
}

func NewRspec 

func NewRspec(r RunnerConfig) Rspec

func (Rspec) CommandNameAndArgs 

func (r Rspec) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)

CommandNameAndArgs replaces the "{{testExamples}}" placeholder in the test command with the test cases. It returns the command name and arguments to run the tests.

func (Rspec) GetExamples 

func (r Rspec) GetExamples(files []string) ([]plan.TestCase, error)

GetExamples returns an array of test examples within the given files.

func (Rspec) GetFiles 

func (r Rspec) GetFiles() ([]string, error)

GetFiles returns an array of file names using the discovery pattern.

func (Rspec) Name 

func (r Rspec) Name() string

func (Rspec) ParseReport 

func (r Rspec) ParseReport(path string) (RspecReport, error)

func (Rspec) ResultFormat added in v2.7.0 

func (r Rspec) ResultFormat() string

func (Rspec) Run 

func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes the test command with the given test cases. If retry is true, it will run the command using the retry test command, otherwise it will use the test command.

Error is returned if the command fails to run, exits prematurely, or if the output cannot be parsed.

Test failure is not considered an error, and is instead returned as a RunResult.

func (Rspec) SupportedFeatures added in v2.7.0 

func (r Rspec) SupportedFeatures() SupportedFeatures

type RspecExample 

type RspecExample struct {
	ID              string  `json:"id"`
	Description     string  `json:"description"`
	FullDescription string  `json:"full_description"`
	Status          string  `json:"status"`
	FilePath        string  `json:"file_path"`
	LineNumber      int     `json:"line_number"`
	RunTime         float64 `json:"run_time"`
}

RspecExample represents a single test example in an Rspec report.

type RspecReport 

type RspecReport struct {
	Version  string         `json:"version"`
	Seed     int            `json:"seed"`
	Examples []RspecExample `json:"examples"`
	Summary  struct {
		ExampleCount                 int `json:"example_count"`
		FailureCount                 int `json:"failure_count"`
		PendingCount                 int `json:"pending_count"`
		ErrorsOutsideOfExamplesCount int `json:"errors_outside_of_examples_count"`
	}
}

RspecReport is the structure for Rspec JSON report.

type RunResult 

type RunResult struct {
	// contains filtered or unexported fields
}

RunResult is a struct to keep track the results of a test run. It contains the logics to record test results, calculate the status of the run.

func NewRunResult 

func NewRunResult(mutedTests []plan.TestCase) *RunResult

func (*RunResult) Error 

func (r *RunResult) Error() error

func (*RunResult) FailedMutedTests 

func (r *RunResult) FailedMutedTests() []plan.TestCase

func (*RunResult) FailedTests 

func (r *RunResult) FailedTests() []plan.TestCase

FailedTests returns a list of test cases that failed.

func (*RunResult) MutedTests 

func (r *RunResult) MutedTests() []TestResult

func (*RunResult) OnlyMutedFailures 

func (r *RunResult) OnlyMutedFailures() bool

func (*RunResult) RecordTestResult 

func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)

RecordTestResult records the result of a test case. If the test case found in the mutedTestLookup, it will be marked as muted.

func (*RunResult) SkippedTests 

func (r *RunResult) SkippedTests() []plan.TestCase

func (*RunResult) Statistics 

func (r *RunResult) Statistics() RunStatistics

func (*RunResult) Status 

func (r *RunResult) Status() RunStatus

Status returns the overall status of the test run. If there is an error, it returns RunStatusError. If there are failed tests, it returns RunStatusFailed. Otherwise, it returns RunStatusPassed.

type RunStatistics 

type RunStatistics struct {
	Total            int `json:"total"`
	PassedOnFirstRun int `json:"passed_on_first_run"`
	PassedOnRetry    int `json:"passed_on_retry"`
	MutedPassed      int `json:"muted_passed"`
	MutedFailed      int `json:"muted_failed"`
	Failed           int `json:"failed"`
	Skipped          int `json:"skipped"`
}

type RunStatus 

type RunStatus string
const (
	// RunStatusPassed indicates that the run was completed and all tests passed.
	RunStatusPassed RunStatus = "passed"
	// RunStatusFailed indicates that the run was completed, but one or more tests failed.
	RunStatusFailed RunStatus = "failed"
	// RunStatusError indicates that the run was completed, but there was an error outside of the tests.
	RunStatusError RunStatus = "error"
	// RunStatusUnknown indicates that the run status is unknown.
	// It could be that no tests were run, run was interrupted, or the report is not available.
	RunStatusUnknown RunStatus = "unknown"
)

type RunnerConfig 

type RunnerConfig struct {
	TestRunner string

	// ResultPath is used internally so bktec can read result from Test Runner.
	// User typically don't need to worry about setting this except in in RSpec and playwright.
	// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
	ResultPath             string
	RetryTestCommand       string
	TagFilters             string
	TestCommand            string
	TestFileExcludePattern string
	TestFilePattern        string
	// contains filtered or unexported fields
}

func (RunnerConfig) LocationPrefix 

func (rc RunnerConfig) LocationPrefix() string

func (RunnerConfig) ResultFilePath added in v2.7.0 

func (rc RunnerConfig) ResultFilePath() string

ResultFilePath returns the path to the runner's raw result file.

func (RunnerConfig) ResultFormat added in v2.7.0 

func (rc RunnerConfig) ResultFormat() string

ResultFormat returns an empty string by default. Runners that support raw result uploads should override this.

func (RunnerConfig) UploadToken added in v2.6.0 

func (rc RunnerConfig) UploadToken() string

type SupportedFeatures added in v2.7.0 

type SupportedFeatures struct {
	SplitByFile     bool
	SplitByExample  bool
	FilterTestFiles bool
	FilterTestByTag bool
	AutoRetry       bool
	Mute            bool
	Skip            bool
}

type TestEngineTest 

type TestEngineTest struct {
	ID       string
	Name     string
	Scope    string
	Location string
	FileName string `json:"file_name,omitempty"`
	Result   TestStatus
}

TestEngineTest represents a Test Engine test result object. Some attributes such as `history` and `failure_reason` are omitted as they are not needed by bktec. Ref: https://buildkite.com/docs/test-engine/importing-json#json-test-results-data-reference-test-result-objects

Currently, only pytest and custom runner uses result from test collector.

type TestResult 

type TestResult struct {
	plan.TestCase
	Status         TestStatus
	ExecutionCount int
	Muted          bool
}

TestResult is a struct to keep track the result of an individual test case.

type TestRunner 

type TestRunner interface {
	Run(result *RunResult, testCases []plan.TestCase, retry bool) error
	GetExamples(files []string) ([]plan.TestCase, error)
	GetFiles() ([]string, error)
	Name() string
	CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
	LocationPrefix() string
	SupportedFeatures() SupportedFeatures
	UploadToken() string
	// ResultFormat returns the format identifier for the runner's raw result file
	// (e.g. "rspec-json"), used when uploading results to Test Engine.
	// Returns an empty string if the runner does not support raw result uploads.
	ResultFormat() string
	// ResultFilePath returns the path to the runner's raw result file.
	ResultFilePath() string
}

func DetectRunner 

func DetectRunner(cfg *config.Config) (TestRunner, error)

type TestStatus 

type TestStatus string
const (
	TestStatusPassed  TestStatus = "passed"
	TestStatusFailed  TestStatus = "failed"
	TestStatusSkipped TestStatus = "skipped"
	TestStatusUnknown TestStatus = "unknown"
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.