Documentation
¶
Overview ¶
Package runner provides the test runners that run sets of test cases. Supported runners: rspec, jest, vitest, cypress, playwright, pytest, gotest, cucumber.
Index ¶
- type Cucumber
- func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (c Cucumber) DiscoverTestTargets() ([]string, error)
- func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error)
- func (c Cucumber) Name() string
- func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)
- func (c Cucumber) ResultFormat() string
- func (c Cucumber) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (c Cucumber) SupportedFeatures() SupportedFeatures
- type CucumberDataTableRow
- type CucumberDocString
- type CucumberElement
- type CucumberFeature
- type CucumberMatch
- type CucumberResult
- type CucumberStep
- type CucumberTag
- type Custom
- func (r Custom) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (r Custom) DiscoverTestTargets() ([]string, error)
- func (r Custom) Name() string
- func (r Custom) ResultFormat() string
- func (r Custom) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (c Custom) SupportedFeatures() SupportedFeatures
- type Cypress
- func (c Cypress) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (c Cypress) DiscoverTestTargets() ([]string, error)
- func (c Cypress) Name() string
- func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (c Cypress) SupportedFeatures() SupportedFeatures
- type ExampleDiscoverer
- type GoTest
- func (g GoTest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (g GoTest) DiscoverTestTargets() ([]string, error)
- func (g GoTest) Name() string
- func (g GoTest) ResultFormat() string
- func (g GoTest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (g GoTest) SupportedFeatures() SupportedFeatures
- type JUnitXMLError
- type JUnitXMLFailure
- type JUnitXMLSkipped
- type JUnitXMLTestCase
- type Jest
- func (j Jest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (j Jest) DiscoverTestTargets() ([]string, error)
- func (j Jest) Name() string
- func (j Jest) ParseReport(path string) (JestReport, error)
- func (j Jest) ResultFormat() string
- func (j Jest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (j Jest) SupportedFeatures() SupportedFeatures
- type JestExample
- type JestReport
- type Playwright
- func (p Playwright) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p Playwright) DiscoverTestTargets() ([]string, error)
- func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)
- func (p Playwright) Name() string
- func (p Playwright) ResultFormat() string
- func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (p Playwright) SupportedFeatures() SupportedFeatures
- type PlaywrightReport
- type PlaywrightReportSuite
- type PlaywrightSpec
- type PlaywrightTest
- type ProcessSignaledError
- type Pytest
- func (p Pytest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p Pytest) DiscoverTestTargets() ([]string, error)
- func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error)
- func (p Pytest) Name() string
- func (p Pytest) ResultFormat() string
- func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (p Pytest) SupportedFeatures() SupportedFeatures
- type Rspec
- func (r Rspec) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (r Rspec) DiscoverTestTargets() ([]string, error)
- func (r Rspec) GetExamples(files []string) ([]plan.TestCase, error)
- func (r Rspec) Name() string
- func (r Rspec) ParseReport(path string) (RspecReport, error)
- func (r Rspec) ResultFormat() string
- func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (r Rspec) SupportedFeatures() SupportedFeatures
- type RspecExample
- type RspecReport
- type RunResult
- func (r *RunResult) Error() error
- func (r *RunResult) FailedMutedTests() []plan.TestCase
- func (r *RunResult) FailedTests() []plan.TestCase
- func (r *RunResult) MutedTests() []TestResult
- func (r *RunResult) OnlyMutedFailures() bool
- func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)
- func (r *RunResult) SkippedTests() []plan.TestCase
- func (r *RunResult) Statistics() RunStatistics
- func (r *RunResult) Status() RunStatus
- type RunStatistics
- type RunStatus
- type RunnerConfig
- type SupportedFeatures
- type TestEngineTest
- type TestResult
- type TestRunner
- type TestRunnerWithTargetDiscovery
- type TestStatus
- type TestTargetDiscoverer
- type Vitest
- func (v Vitest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (v Vitest) DiscoverTestTargets() ([]string, error)
- func (v Vitest) Name() string
- func (v Vitest) ResultFormat() string
- func (v Vitest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- func (v Vitest) SupportedFeatures() SupportedFeatures
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Cucumber ¶
type Cucumber struct {
RunnerConfig
}
func NewCucumber ¶
func NewCucumber(c RunnerConfig) Cucumber
func (Cucumber) CommandNameAndArgs ¶
func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
CommandNameAndArgs replaces placeholders and returns command + args.
func (Cucumber) DiscoverTestTargets ¶
DiscoverTestTargets returns feature files based on include / exclude pattern.
func (Cucumber) GetExamples ¶
GetExamples returns an array of test scenarios within the given feature files.
func (Cucumber) ParseReport ¶
func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)
func (Cucumber) ResultFormat ¶
func (Cucumber) SupportedFeatures ¶
func (c Cucumber) SupportedFeatures() SupportedFeatures
type CucumberDataTableRow ¶
type CucumberDataTableRow struct {
Cells []string `json:"cells"`
}
CucumberDataTableRow represents a row in a step's data table.
type CucumberDocString ¶
type CucumberDocString struct {
Value string `json:"value"`
ContentType string `json:"content_type"`
Line int `json:"line"`
}
CucumberDocString represents a doc string argument for a step.
type CucumberElement ¶
type CucumberElement struct {
ID string `json:"id"`
Keyword string `json:"keyword"`
Name string `json:"name"`
Description string `json:"description"`
Line int `json:"line"`
Type string `json:"type"` // e.g., "scenario", "background"
Steps []CucumberStep `json:"steps"`
Tags []CucumberTag `json:"tags,omitempty"`
}
CucumberElement represents a scenario or background in Cucumber's JSON output.
func (CucumberElement) AggregatedStatus ¶
func (e CucumberElement) AggregatedStatus() string
AggregatedStatus returns overall scenario status based on its steps. It mirrors the logic previously in cucumber.go but uses the parser's structs.
type CucumberFeature ¶
type CucumberFeature struct {
URI string `json:"uri"`
ID string `json:"id"`
Keyword string `json:"keyword"`
Name string `json:"name"`
Description string `json:"description"`
Line int `json:"line"`
Elements []CucumberElement `json:"elements"`
Tags []CucumberTag `json:"tags,omitempty"`
}
CucumberFeature represents a single feature in Cucumber's JSON output.
func ParseCucumberJSONReport ¶
func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error)
ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). This is for actual test results.
type CucumberMatch ¶
type CucumberMatch struct {
Location string `json:"location"` // e.g., "features/step_definitions/steps.rb:5"
}
CucumberMatch represents the matching step definition for a step.
type CucumberResult ¶
type CucumberResult struct {
Status string `json:"status"` // e.g., "passed", "failed", "skipped", "undefined"
ErrorMessage string `json:"error_message,omitempty"`
Duration int64 `json:"duration,omitempty"` // Nanoseconds
}
CucumberResult represents the result of a step execution.
type CucumberStep ¶
type CucumberStep struct {
Keyword string `json:"keyword"`
Name string `json:"name"`
Line int `json:"line"`
Result *CucumberResult `json:"result,omitempty"`
Match *CucumberMatch `json:"match,omitempty"`
DocString *CucumberDocString `json:"doc_string,omitempty"`
DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables
}
CucumberStep represents a single step in a scenario.
type CucumberTag ¶
CucumberTag represents a tag in Cucumber.
type Custom ¶
type Custom struct {
RunnerConfig
}
func NewCustom ¶
func NewCustom(r RunnerConfig) (Custom, error)
func (Custom) CommandNameAndArgs ¶
func (Custom) DiscoverTestTargets ¶
DiscoverTestTargets returns file names using the discovery pattern. Custom selector splitting without a selector list falls back to file discovery.
func (Custom) ResultFormat ¶
func (Custom) SupportedFeatures ¶
func (c Custom) SupportedFeatures() SupportedFeatures
type Cypress ¶
type Cypress struct {
RunnerConfig
}
func NewCypress ¶
func NewCypress(c RunnerConfig) Cypress
func (Cypress) CommandNameAndArgs ¶
func (Cypress) DiscoverTestTargets ¶
func (Cypress) SupportedFeatures ¶
func (c Cypress) SupportedFeatures() SupportedFeatures
type ExampleDiscoverer ¶
type GoTest ¶
type GoTest struct {
RunnerConfig
// contains filtered or unexported fields
}
func NewGoTest ¶
func NewGoTest(c RunnerConfig) GoTest
func (GoTest) CommandNameAndArgs ¶
func (GoTest) DiscoverTestTargets ¶
DiscoverTestTargets discovers the Go packages that contain tests and returns their import paths to be used as discovery targets.
"File" does not exist as a first level concept in Go projects, so a package import path is the smallest unit we can split on. The returned import paths are sent to the test plan API as selector values.
We use a format template so `go list` only emits packages that actually have test files (_test.go). Packages with zero tests would otherwise be sent to the test plan API as split units that run no tests, taking up a bin packing slot for nothing.
func (GoTest) ResultFormat ¶
func (GoTest) SupportedFeatures ¶
func (g GoTest) SupportedFeatures() SupportedFeatures
type JUnitXMLError ¶
type JUnitXMLError struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Content string `xml:",chardata"`
}
JUnitXMLError represents the <error> element in JUnit XML
type JUnitXMLFailure ¶
type JUnitXMLFailure struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Content string `xml:",chardata"`
}
JUnitXMLFailure represents the <failure> element in JUnit XML
type JUnitXMLSkipped ¶
type JUnitXMLSkipped struct {
Message string `xml:"message,attr"`
}
JUnitXMLSkipped represents the <skipped> element in JUnit XML
type JUnitXMLTestCase ¶
type JUnitXMLTestCase struct {
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Result TestStatus // passed | failed | skipped
Failure *JUnitXMLFailure `xml:"failure"`
Error *JUnitXMLError `xml:"error"`
Skipped *JUnitXMLSkipped `xml:"skipped"`
// SuiteName is the name attribute of the enclosing <testsuite> element.
SuiteName string `xml:"-"`
}
JUnitXMLTestCase represents a single <testcase> element in JUnit XML.
type Jest ¶
type Jest struct {
RunnerConfig
}
func NewJest ¶
func NewJest(j RunnerConfig) Jest
func (Jest) CommandNameAndArgs ¶
func (Jest) DiscoverTestTargets ¶
DiscoverTestTargets returns file names using the discovery pattern.
func (Jest) ParseReport ¶
func (j Jest) ParseReport(path string) (JestReport, error)
func (Jest) ResultFormat ¶
func (Jest) SupportedFeatures ¶
func (j Jest) SupportedFeatures() SupportedFeatures
type JestExample ¶
type JestReport ¶
type JestReport struct {
NumFailedTests int
TestResults []struct {
AssertionResults []JestExample
FileName string `json:"name"`
Status string
}
}
func ParseJestReport ¶
func ParseJestReport(path string) (JestReport, error)
ParseJestReport parses a Jest-compatible JSON report file. It is shared by the Jest and Vitest runners, since Vitest's JSON reporter emits the same shape.
type Playwright ¶
type Playwright struct {
RunnerConfig
}
func NewPlaywright ¶
func NewPlaywright(p RunnerConfig) Playwright
func (Playwright) CommandNameAndArgs ¶
func (Playwright) DiscoverTestTargets ¶
func (p Playwright) DiscoverTestTargets() ([]string, error)
func (Playwright) GetExamples ¶
func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)
GetExamples returns an array of test examples within the given files.
func (Playwright) Name ¶
func (p Playwright) Name() string
func (Playwright) ResultFormat ¶
func (p Playwright) ResultFormat() string
func (Playwright) SupportedFeatures ¶
func (p Playwright) SupportedFeatures() SupportedFeatures
type PlaywrightReport ¶
type PlaywrightReport struct {
Suites []PlaywrightReportSuite
Stats struct {
Expected int
Unexpected int
}
Errors []struct {
Message string
}
}
type PlaywrightReportSuite ¶
type PlaywrightReportSuite struct {
Title string
Specs []PlaywrightSpec
Suites []PlaywrightReportSuite
}
type PlaywrightSpec ¶
type PlaywrightTest ¶
type ProcessSignaledError ¶
func (*ProcessSignaledError) Error ¶
func (e *ProcessSignaledError) Error() string
type Pytest ¶
type Pytest struct {
RunnerConfig
// contains filtered or unexported fields
}
func NewPytest ¶
func NewPytest(c RunnerConfig) Pytest
func (Pytest) CommandNameAndArgs ¶
func (Pytest) DiscoverTestTargets ¶
func (Pytest) GetExamples ¶
GetExamples returns an array of test examples within the given files. It uses `pytest --collect-only -q` to enumerate individual tests.
--tag-filters can be used to filter tests by markers if specified. e.g. --tag-fitlers team:frontend matches markers: with @pytest.mark.execution_tag('team', 'frontend')
The --tag-filters feature also assumes Python Test Collector plugin version >1.2.0 is installed.
func (Pytest) ResultFormat ¶
func (Pytest) SupportedFeatures ¶
func (p Pytest) SupportedFeatures() SupportedFeatures
type Rspec ¶
type Rspec struct {
RunnerConfig
}
func NewRspec ¶
func NewRspec(r RunnerConfig) Rspec
func (Rspec) CommandNameAndArgs ¶
CommandNameAndArgs replaces the "{{testExamples}}" placeholder in the test command with the test cases. It returns the command name and arguments to run the tests.
func (Rspec) DiscoverTestTargets ¶
DiscoverTestTargets returns file names using the discovery pattern.
func (Rspec) GetExamples ¶
GetExamples returns an array of test examples within the given files.
func (Rspec) ParseReport ¶
func (r Rspec) ParseReport(path string) (RspecReport, error)
func (Rspec) ResultFormat ¶
func (Rspec) Run ¶
Run executes the test command with the given test cases. If retry is true, it will run the command using the retry test command, otherwise it will use the test command.
Error is returned if the command fails to run, exits prematurely, or if the output cannot be parsed.
Test failure is not considered an error, and is instead returned as a RunResult.
func (Rspec) SupportedFeatures ¶
func (r Rspec) SupportedFeatures() SupportedFeatures
type RspecExample ¶
type RspecExample struct {
ID string `json:"id"`
Description string `json:"description"`
FullDescription string `json:"full_description"`
Status string `json:"status"`
FilePath string `json:"file_path"`
LineNumber int `json:"line_number"`
RunTime float64 `json:"run_time"`
}
RspecExample represents a single test example in an Rspec report.
type RspecReport ¶
type RspecReport struct {
Version string `json:"version"`
Seed int `json:"seed"`
Examples []RspecExample `json:"examples"`
Summary struct {
ExampleCount int `json:"example_count"`
FailureCount int `json:"failure_count"`
PendingCount int `json:"pending_count"`
ErrorsOutsideOfExamplesCount int `json:"errors_outside_of_examples_count"`
}
}
RspecReport is the structure for Rspec JSON report.
type RunResult ¶
type RunResult struct {
// contains filtered or unexported fields
}
RunResult is a struct to keep track the results of a test run. It contains the logics to record test results, calculate the status of the run.
func NewRunResult ¶
func (*RunResult) FailedMutedTests ¶
func (*RunResult) FailedTests ¶
FailedTests returns a list of test cases that failed.
func (*RunResult) MutedTests ¶
func (r *RunResult) MutedTests() []TestResult
func (*RunResult) OnlyMutedFailures ¶
func (*RunResult) RecordTestResult ¶
func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)
RecordTestResult records the result of a test case. If the test case found in the mutedTestLookup, it will be marked as muted.
func (*RunResult) SkippedTests ¶
func (*RunResult) Statistics ¶
func (r *RunResult) Statistics() RunStatistics
type RunStatistics ¶
type RunStatus ¶
type RunStatus string
const ( // RunStatusPassed indicates that the run was completed and all tests passed. RunStatusPassed RunStatus = "passed" // RunStatusFailed indicates that the run was completed, but one or more tests failed. RunStatusFailed RunStatus = "failed" // RunStatusError indicates that the run was completed, but there was an error outside of the tests. RunStatusError RunStatus = "error" // RunStatusUnknown indicates that the run status is unknown. // It could be that no tests were run, run was interrupted, or the report is not available. RunStatusUnknown RunStatus = "unknown" )
type RunnerConfig ¶
type RunnerConfig struct {
TestRunner string
// ResultPath is used internally so bktec can read result from Test Runner.
// User typically don't need to worry about setting this except in in RSpec and playwright.
// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
ResultPath string
RetryTestCommand string
TagFilters string
TestCommand string
TestFileExcludePattern string
TestFilePattern string
// SelectorListPath points at a file containing the selectors to run.
SelectorListPath string
// contains filtered or unexported fields
}
func (RunnerConfig) LocationPrefix ¶
func (rc RunnerConfig) LocationPrefix() string
func (RunnerConfig) ResultFilePath ¶
func (rc RunnerConfig) ResultFilePath() string
ResultFilePath returns the path to the runner's raw result file.
func (RunnerConfig) ResultFormat ¶
func (rc RunnerConfig) ResultFormat() string
ResultFormat returns an empty string by default. Runners that support raw result uploads should override this.
func (RunnerConfig) UploadToken ¶
func (rc RunnerConfig) UploadToken() string
type SupportedFeatures ¶
type TestEngineTest ¶
type TestEngineTest struct {
ID string
Name string
Scope string
Location string
FileName string `json:"file_name,omitempty"`
Result TestStatus
Tags map[string]string `json:"tags,omitempty"`
}
TestEngineTest represents a Test Engine test result object. Some attributes such as `history` and `failure_reason` are omitted as they are not needed by bktec. Ref: https://buildkite.com/docs/test-engine/importing-json#json-test-results-data-reference-test-result-objects
Currently, only pytest and custom runner uses result from test collector.
type TestResult ¶
type TestResult struct {
plan.TestCase
Status TestStatus
ExecutionCount int
Muted bool
}
TestResult is a struct to keep track the result of an individual test case.
type TestRunner ¶
type TestRunner interface {
Run(result *RunResult, testCases []plan.TestCase, retry bool) error
Name() string
CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
LocationPrefix() string
SupportedFeatures() SupportedFeatures
UploadToken() string
// ResultFormat returns the format identifier for the runner's raw result file
// (e.g. "rspec-json"), used when uploading results to Test Engine.
// Returns an empty string if the runner does not support raw result uploads.
ResultFormat() string
// ResultFilePath returns the path to the runner's raw result file.
ResultFilePath() string
}
type TestRunnerWithTargetDiscovery ¶
type TestRunnerWithTargetDiscovery interface {
TestRunner
TestTargetDiscoverer
}
func DetectRunner ¶
func DetectRunner(cfg *config.Config) (TestRunnerWithTargetDiscovery, error)
type TestStatus ¶
type TestStatus string
const ( TestStatusPassed TestStatus = "passed" TestStatusFailed TestStatus = "failed" TestStatusSkipped TestStatus = "skipped" TestStatusUnknown TestStatus = "unknown" )
type TestTargetDiscoverer ¶
type Vitest ¶
type Vitest struct {
RunnerConfig
}
Vitest is a first-class runner for the Vitest test framework.
Vitest's JSON reporter is Jest-compatible, so the result file is parsed using the shared JestReport types. The main differences from the Jest runner are the default test/retry commands and the result format identifier.
func NewVitest ¶
func NewVitest(v RunnerConfig) Vitest
func (Vitest) CommandNameAndArgs ¶
func (Vitest) DiscoverTestTargets ¶
DiscoverTestTargets returns file names using the discovery pattern.
func (Vitest) ResultFormat ¶
func (Vitest) SupportedFeatures ¶
func (v Vitest) SupportedFeatures() SupportedFeatures