Documentation
¶
Overview ¶
Package runner provides the test runners that run sets of test cases. Supported runners: rspec, jest, vitest, cypress, playwright, pytest, gotest, cucumber.
Index ¶
- type Cucumber
- func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error)
- func (c Cucumber) GetFiles() ([]string, error)
- func (c Cucumber) GetSelectors() ([]string, error)
- func (c Cucumber) Name() string
- func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)
- func (c Cucumber) ResultFormat() string
- func (c Cucumber) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (c Cucumber) SupportedFeatures() SupportedFeatures
- type CucumberDataTableRow
- type CucumberDocString
- type CucumberElement
- type CucumberFeature
- type CucumberMatch
- type CucumberResult
- type CucumberStep
- type CucumberTag
- type Custom
- func (r Custom) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (r Custom) GetExamples(files []string) ([]plan.TestCase, error)
- func (r Custom) GetFiles() ([]string, error)
- func (r Custom) GetSelectors() ([]string, error)
- func (r Custom) Name() string
- func (r Custom) ResultFormat() string
- func (r Custom) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (c Custom) SupportedFeatures() SupportedFeatures
- type Cypress
- func (c Cypress) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (c Cypress) GetExamples(files []string) ([]plan.TestCase, error)
- func (c Cypress) GetFiles() ([]string, error)
- func (c Cypress) GetSelectors() ([]string, error)
- func (c Cypress) Name() string
- func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (c Cypress) SupportedFeatures() SupportedFeatures
- type GoTest
- func (g GoTest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (g GoTest) GetExamples(files []string) ([]plan.TestCase, error)
- func (g GoTest) GetFiles() ([]string, error)
- func (g GoTest) GetSelectors() ([]string, error)
- func (g GoTest) Name() string
- func (g GoTest) ResultFormat() string
- func (g GoTest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (g GoTest) SupportedFeatures() SupportedFeatures
- type JUnitXMLError
- type JUnitXMLFailure
- type JUnitXMLSkipped
- type JUnitXMLTestCase
- type Jest
- func (j Jest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (j Jest) GetExamples(files []string) ([]plan.TestCase, error)
- func (j Jest) GetFiles() ([]string, error)
- func (j Jest) GetSelectors() ([]string, error)
- func (j Jest) Name() string
- func (j Jest) ParseReport(path string) (JestReport, error)
- func (j Jest) ResultFormat() string
- func (j Jest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (j Jest) SupportedFeatures() SupportedFeatures
- type JestExample
- type JestReport
- type Playwright
- func (p Playwright) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)
- func (p Playwright) GetFiles() ([]string, error)
- func (p Playwright) GetSelectors() ([]string, error)
- func (p Playwright) Name() string
- func (p Playwright) ResultFormat() string
- func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (p Playwright) SupportedFeatures() SupportedFeatures
- type PlaywrightReport
- type PlaywrightReportSuite
- type PlaywrightSpec
- type PlaywrightTest
- type ProcessSignaledError
- type Pytest
- func (p Pytest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error)
- func (p Pytest) GetFiles() ([]string, error)
- func (p Pytest) GetSelectors() ([]string, error)
- func (p Pytest) Name() string
- func (p Pytest) ResultFormat() string
- func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (p Pytest) SupportedFeatures() SupportedFeatures
- type Rspec
- func (r Rspec) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (r Rspec) GetExamples(files []string) ([]plan.TestCase, error)
- func (r Rspec) GetFiles() ([]string, error)
- func (r Rspec) GetSelectors() ([]string, error)
- func (r Rspec) Name() string
- func (r Rspec) ParseReport(path string) (RspecReport, error)
- func (r Rspec) ResultFormat() string
- func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (r Rspec) SupportedFeatures() SupportedFeatures
- type RspecExample
- type RspecReport
- type RunResult
- func (r *RunResult) Error() error
- func (r *RunResult) FailedMutedTests() []plan.TestCase
- func (r *RunResult) FailedTests() []plan.TestCase
- func (r *RunResult) MutedTests() []TestResult
- func (r *RunResult) OnlyMutedFailures() bool
- func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)
- func (r *RunResult) SkippedTests() []plan.TestCase
- func (r *RunResult) Statistics() RunStatistics
- func (r *RunResult) Status() RunStatus
- type RunStatistics
- type RunStatus
- type RunnerConfig
- type SupportedFeatures
- type TestEngineTest
- type TestResult
- type TestRunner
- type TestStatus
- type Vitest
- func (v Vitest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (v Vitest) GetExamples(files []string) ([]plan.TestCase, error)
- func (v Vitest) GetFiles() ([]string, error)
- func (v Vitest) GetSelectors() ([]string, error)
- func (v Vitest) Name() string
- func (v Vitest) ResultFormat() string
- func (v Vitest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (v Vitest) SupportedFeatures() SupportedFeatures
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Cucumber ¶
type Cucumber struct {
RunnerConfig
}
func NewCucumber ¶
func NewCucumber(c RunnerConfig) Cucumber
func (Cucumber) CommandNameAndArgs ¶
func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
CommandNameAndArgs replaces placeholders and returns command + args.
func (Cucumber) GetExamples ¶
GetExamples returns an array of test scenarios within the given feature files.
func (Cucumber) GetFiles ¶
GetFiles returns the list of feature files based on include / exclude pattern.
func (Cucumber) GetSelectors ¶ added in v2.10.0
func (Cucumber) ParseReport ¶
func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)
func (Cucumber) ResultFormat ¶ added in v2.7.0
func (Cucumber) SupportedFeatures ¶ added in v2.7.0
func (c Cucumber) SupportedFeatures() SupportedFeatures
type CucumberDataTableRow ¶
type CucumberDataTableRow struct {
Cells []string `json:"cells"`
}
CucumberDataTableRow represents a row in a step's data table.
type CucumberDocString ¶
type CucumberDocString struct {
Value string `json:"value"`
ContentType string `json:"content_type"`
Line int `json:"line"`
}
CucumberDocString represents a doc string argument for a step.
type CucumberElement ¶
type CucumberElement struct {
ID string `json:"id"`
Keyword string `json:"keyword"`
Name string `json:"name"`
Description string `json:"description"`
Line int `json:"line"`
Type string `json:"type"` // e.g., "scenario", "background"
Steps []CucumberStep `json:"steps"`
Tags []CucumberTag `json:"tags,omitempty"`
}
CucumberElement represents a scenario or background in Cucumber's JSON output.
func (CucumberElement) AggregatedStatus ¶
func (e CucumberElement) AggregatedStatus() string
AggregatedStatus returns overall scenario status based on its steps. It mirrors the logic previously in cucumber.go but uses the parser's structs.
type CucumberFeature ¶
type CucumberFeature struct {
URI string `json:"uri"`
ID string `json:"id"`
Keyword string `json:"keyword"`
Name string `json:"name"`
Description string `json:"description"`
Line int `json:"line"`
Elements []CucumberElement `json:"elements"`
Tags []CucumberTag `json:"tags,omitempty"`
}
CucumberFeature represents a single feature in Cucumber's JSON output.
func ParseCucumberJSONReport ¶
func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error)
ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). This is for actual test results.
type CucumberMatch ¶
type CucumberMatch struct {
Location string `json:"location"` // e.g., "features/step_definitions/steps.rb:5"
}
CucumberMatch represents the matching step definition for a step.
type CucumberResult ¶
type CucumberResult struct {
Status string `json:"status"` // e.g., "passed", "failed", "skipped", "undefined"
ErrorMessage string `json:"error_message,omitempty"`
Duration int64 `json:"duration,omitempty"` // Nanoseconds
}
CucumberResult represents the result of a step execution.
type CucumberStep ¶
type CucumberStep struct {
Keyword string `json:"keyword"`
Name string `json:"name"`
Line int `json:"line"`
Result *CucumberResult `json:"result,omitempty"`
Match *CucumberMatch `json:"match,omitempty"`
DocString *CucumberDocString `json:"doc_string,omitempty"`
DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables
}
CucumberStep represents a single step in a scenario.
type CucumberTag ¶
CucumberTag represents a tag in Cucumber.
type Custom ¶
type Custom struct {
RunnerConfig
}
func NewCustom ¶
func NewCustom(r RunnerConfig) (Custom, error)
func (Custom) CommandNameAndArgs ¶
func (Custom) GetSelectors ¶ added in v2.9.0
GetSelectors falls back to file-pattern discovery, since the custom runner has no way to discover selectors on its own without a selector file (in which case bktec reads selectors from the file directly, without calling this).
func (Custom) ResultFormat ¶ added in v2.7.0
func (Custom) SupportedFeatures ¶ added in v2.7.0
func (c Custom) SupportedFeatures() SupportedFeatures
type Cypress ¶
type Cypress struct {
RunnerConfig
}
func NewCypress ¶
func NewCypress(c RunnerConfig) Cypress
func (Cypress) CommandNameAndArgs ¶
func (Cypress) GetSelectors ¶ added in v2.10.0
func (Cypress) SupportedFeatures ¶ added in v2.7.0
func (c Cypress) SupportedFeatures() SupportedFeatures
type GoTest ¶
type GoTest struct {
RunnerConfig
// contains filtered or unexported fields
}
func NewGoTest ¶
func NewGoTest(c RunnerConfig) GoTest
func (GoTest) CommandNameAndArgs ¶
func (GoTest) GetFiles ¶
GetFiles is the fallback used when selector based splitting is disabled, which is the default behavior. Because "file" does not exist as a first level concept in Go projects, there is nothing file-like to discover, so we reuse the same package discovery as GetSelectors and return package import paths instead.
The implication is that server side smart test splitting will never work for these targets, since the API treats them as file paths. It almost always falls back to simple splitting. We are migrating Go splitting over to GetSelectors so the API can split by selector properly.
func (GoTest) GetSelectors ¶ added in v2.9.0
GetSelectors discovers the Go packages that contain tests and returns their import paths to be used as selectors for selector based splitting.
"File" does not exist as a first level concept in Go projects, so a package import path is the smallest unit we can split on. The returned import paths are sent to the test plan API as selector values.
We use a format template so `go list` only emits packages that actually have test files (_test.go). Packages with zero tests would otherwise be sent to the test plan API as split units that run no tests, taking up a bin packing slot for nothing.
func (GoTest) ResultFormat ¶ added in v2.7.0
func (GoTest) SupportedFeatures ¶ added in v2.7.0
func (g GoTest) SupportedFeatures() SupportedFeatures
type JUnitXMLError ¶ added in v2.7.0
type JUnitXMLError struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Content string `xml:",chardata"`
}
JUnitXMLError represents the <error> element in JUnit XML
type JUnitXMLFailure ¶
type JUnitXMLFailure struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Content string `xml:",chardata"`
}
JUnitXMLFailure represents the <failure> element in JUnit XML
type JUnitXMLSkipped ¶
type JUnitXMLSkipped struct {
Message string `xml:"message,attr"`
}
JUnitXMLSkipped represents the <skipped> element in JUnit XML
type JUnitXMLTestCase ¶ added in v2.7.0
type JUnitXMLTestCase struct {
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Result TestStatus // passed | failed | skipped
Failure *JUnitXMLFailure `xml:"failure"`
Error *JUnitXMLError `xml:"error"`
Skipped *JUnitXMLSkipped `xml:"skipped"`
// SuiteName is the name attribute of the enclosing <testsuite> element.
SuiteName string `xml:"-"`
}
JUnitXMLTestCase represents a single <testcase> element in JUnit XML.
type Jest ¶
type Jest struct {
RunnerConfig
}
func NewJest ¶
func NewJest(j RunnerConfig) Jest
func (Jest) CommandNameAndArgs ¶
func (Jest) GetSelectors ¶ added in v2.10.0
func (Jest) ParseReport ¶
func (j Jest) ParseReport(path string) (JestReport, error)
func (Jest) ResultFormat ¶ added in v2.7.0
func (Jest) SupportedFeatures ¶ added in v2.7.0
func (j Jest) SupportedFeatures() SupportedFeatures
type JestExample ¶
type JestReport ¶
type JestReport struct {
NumFailedTests int
TestResults []struct {
AssertionResults []JestExample
FileName string `json:"name"`
Status string
}
}
func ParseJestReport ¶ added in v2.10.0
func ParseJestReport(path string) (JestReport, error)
ParseJestReport parses a Jest-compatible JSON report file. It is shared by the Jest and Vitest runners, since Vitest's JSON reporter emits the same shape.
type Playwright ¶
type Playwright struct {
RunnerConfig
}
func NewPlaywright ¶
func NewPlaywright(p RunnerConfig) Playwright
func (Playwright) CommandNameAndArgs ¶
func (Playwright) GetExamples ¶
func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)
GetExamples returns an array of test examples within the given files.
func (Playwright) GetFiles ¶
func (p Playwright) GetFiles() ([]string, error)
func (Playwright) GetSelectors ¶ added in v2.10.0
func (p Playwright) GetSelectors() ([]string, error)
func (Playwright) Name ¶
func (p Playwright) Name() string
func (Playwright) ResultFormat ¶ added in v2.7.0
func (p Playwright) ResultFormat() string
func (Playwright) SupportedFeatures ¶ added in v2.7.0
func (p Playwright) SupportedFeatures() SupportedFeatures
type PlaywrightReport ¶
type PlaywrightReport struct {
Suites []PlaywrightReportSuite
Stats struct {
Expected int
Unexpected int
}
Errors []struct {
Message string
}
}
type PlaywrightReportSuite ¶
type PlaywrightReportSuite struct {
Title string
Specs []PlaywrightSpec
Suites []PlaywrightReportSuite
}
type PlaywrightSpec ¶
type PlaywrightTest ¶
type ProcessSignaledError ¶
func (*ProcessSignaledError) Error ¶
func (e *ProcessSignaledError) Error() string
type Pytest ¶
type Pytest struct {
RunnerConfig
// contains filtered or unexported fields
}
func NewPytest ¶
func NewPytest(c RunnerConfig) Pytest
func (Pytest) CommandNameAndArgs ¶
func (Pytest) GetExamples ¶
GetExamples returns an array of test examples within the given files. It uses `pytest --collect-only -q` to enumerate individual tests.
--tag-filters can be used to filter tests by markers if specified. e.g. --tag-fitlers team:frontend matches markers: with @pytest.mark.execution_tag('team', 'frontend')
The --tag-filters feature also assumes Python Test Collector plugin version >1.2.0 is installed.
func (Pytest) GetSelectors ¶ added in v2.10.0
func (Pytest) ResultFormat ¶ added in v2.7.0
func (Pytest) SupportedFeatures ¶ added in v2.7.0
func (p Pytest) SupportedFeatures() SupportedFeatures
type Rspec ¶
type Rspec struct {
RunnerConfig
}
func NewRspec ¶
func NewRspec(r RunnerConfig) Rspec
func (Rspec) CommandNameAndArgs ¶
CommandNameAndArgs replaces the "{{testExamples}}" placeholder in the test command with the test cases. It returns the command name and arguments to run the tests.
func (Rspec) GetExamples ¶
GetExamples returns an array of test examples within the given files.
func (Rspec) GetSelectors ¶ added in v2.9.0
func (Rspec) ParseReport ¶
func (r Rspec) ParseReport(path string) (RspecReport, error)
func (Rspec) ResultFormat ¶ added in v2.7.0
func (Rspec) Run ¶
Run executes the test command with the given test cases. If retry is true, it will run the command using the retry test command, otherwise it will use the test command.
Error is returned if the command fails to run, exits prematurely, or if the output cannot be parsed.
Test failure is not considered an error, and is instead returned as a RunResult.
func (Rspec) SupportedFeatures ¶ added in v2.7.0
func (r Rspec) SupportedFeatures() SupportedFeatures
type RspecExample ¶
type RspecExample struct {
ID string `json:"id"`
Description string `json:"description"`
FullDescription string `json:"full_description"`
Status string `json:"status"`
FilePath string `json:"file_path"`
LineNumber int `json:"line_number"`
RunTime float64 `json:"run_time"`
}
RspecExample represents a single test example in an Rspec report.
type RspecReport ¶
type RspecReport struct {
Version string `json:"version"`
Seed int `json:"seed"`
Examples []RspecExample `json:"examples"`
Summary struct {
ExampleCount int `json:"example_count"`
FailureCount int `json:"failure_count"`
PendingCount int `json:"pending_count"`
ErrorsOutsideOfExamplesCount int `json:"errors_outside_of_examples_count"`
}
}
RspecReport is the structure for Rspec JSON report.
type RunResult ¶
type RunResult struct {
// contains filtered or unexported fields
}
RunResult is a struct to keep track the results of a test run. It contains the logics to record test results, calculate the status of the run.
func NewRunResult ¶
func (*RunResult) FailedMutedTests ¶
func (*RunResult) FailedTests ¶
FailedTests returns a list of test cases that failed.
func (*RunResult) MutedTests ¶
func (r *RunResult) MutedTests() []TestResult
func (*RunResult) OnlyMutedFailures ¶
func (*RunResult) RecordTestResult ¶
func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)
RecordTestResult records the result of a test case. If the test case found in the mutedTestLookup, it will be marked as muted.
func (*RunResult) SkippedTests ¶
func (*RunResult) Statistics ¶
func (r *RunResult) Statistics() RunStatistics
type RunStatistics ¶
type RunStatus ¶
type RunStatus string
const ( // RunStatusPassed indicates that the run was completed and all tests passed. RunStatusPassed RunStatus = "passed" // RunStatusFailed indicates that the run was completed, but one or more tests failed. RunStatusFailed RunStatus = "failed" // RunStatusError indicates that the run was completed, but there was an error outside of the tests. RunStatusError RunStatus = "error" // RunStatusUnknown indicates that the run status is unknown. // It could be that no tests were run, run was interrupted, or the report is not available. RunStatusUnknown RunStatus = "unknown" )
type RunnerConfig ¶
type RunnerConfig struct {
TestRunner string
// ResultPath is used internally so bktec can read result from Test Runner.
// User typically don't need to worry about setting this except in in RSpec and playwright.
// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
ResultPath string
RetryTestCommand string
TagFilters string
TestCommand string
TestFileExcludePattern string
TestFilePattern string
// SelectorSplitting enables selector-based splitting for this runner.
SelectorSplitting bool
// SelectorListPath points at a file containing the selectors to run.
SelectorListPath string
// contains filtered or unexported fields
}
func (RunnerConfig) GetSelectors ¶ added in v2.9.0
func (rc RunnerConfig) GetSelectors() ([]string, error)
GetSelectors returns a "not supported" error by default. Runners that support selector based splitting override this. Every other runner inherits this default until it gains selector support.
func (RunnerConfig) LocationPrefix ¶
func (rc RunnerConfig) LocationPrefix() string
func (RunnerConfig) ResultFilePath ¶ added in v2.7.0
func (rc RunnerConfig) ResultFilePath() string
ResultFilePath returns the path to the runner's raw result file.
func (RunnerConfig) ResultFormat ¶ added in v2.7.0
func (rc RunnerConfig) ResultFormat() string
ResultFormat returns an empty string by default. Runners that support raw result uploads should override this.
func (RunnerConfig) UploadToken ¶ added in v2.6.0
func (rc RunnerConfig) UploadToken() string
type SupportedFeatures ¶ added in v2.7.0
type TestEngineTest ¶
type TestEngineTest struct {
ID string
Name string
Scope string
Location string
FileName string `json:"file_name,omitempty"`
Result TestStatus
Tags map[string]string `json:"tags,omitempty"`
}
TestEngineTest represents a Test Engine test result object. Some attributes such as `history` and `failure_reason` are omitted as they are not needed by bktec. Ref: https://buildkite.com/docs/test-engine/importing-json#json-test-results-data-reference-test-result-objects
Currently, only pytest and custom runner uses result from test collector.
type TestResult ¶
type TestResult struct {
plan.TestCase
Status TestStatus
ExecutionCount int
Muted bool
}
TestResult is a struct to keep track the result of an individual test case.
type TestRunner ¶
type TestRunner interface {
Run(result *RunResult, testCases []plan.TestCase, retry bool) error
GetExamples(files []string) ([]plan.TestCase, error)
GetFiles() ([]string, error)
GetSelectors() ([]string, error)
Name() string
CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
LocationPrefix() string
SupportedFeatures() SupportedFeatures
UploadToken() string
// ResultFormat returns the format identifier for the runner's raw result file
// (e.g. "rspec-json"), used when uploading results to Test Engine.
// Returns an empty string if the runner does not support raw result uploads.
ResultFormat() string
// ResultFilePath returns the path to the runner's raw result file.
ResultFilePath() string
}
func DetectRunner ¶
func DetectRunner(cfg *config.Config) (TestRunner, error)
type TestStatus ¶
type TestStatus string
const ( TestStatusPassed TestStatus = "passed" TestStatusFailed TestStatus = "failed" TestStatusSkipped TestStatus = "skipped" TestStatusUnknown TestStatus = "unknown" )
type Vitest ¶ added in v2.10.0
type Vitest struct {
RunnerConfig
}
Vitest is a first-class runner for the Vitest test framework.
Vitest's JSON reporter is Jest-compatible, so the result file is parsed using the shared JestReport types. The main differences from the Jest runner are the default test/retry commands and the result format identifier.
func NewVitest ¶ added in v2.10.0
func NewVitest(v RunnerConfig) Vitest
func (Vitest) CommandNameAndArgs ¶ added in v2.10.0
func (Vitest) GetExamples ¶ added in v2.10.0
func (Vitest) GetFiles ¶ added in v2.10.0
GetFiles returns an array of file names using the discovery pattern.
func (Vitest) GetSelectors ¶ added in v2.10.0
func (Vitest) ResultFormat ¶ added in v2.10.0
func (Vitest) SupportedFeatures ¶ added in v2.10.0
func (v Vitest) SupportedFeatures() SupportedFeatures