Documentation
¶
Overview ¶
Package runner provides the test runners that run sets of test cases. Supported runners: rspec, jest, cypress, playwright, pytest, gotest, cucumber, nunit.
Index ¶
- type Cucumber
- func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error)
- func (c Cucumber) GetFiles() ([]string, error)
- func (c Cucumber) Name() string
- func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)
- func (c Cucumber) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type CucumberDataTableRow
- type CucumberDocString
- type CucumberElement
- type CucumberFeature
- type CucumberMatch
- type CucumberResult
- type CucumberStep
- type CucumberTag
- type Custom
- func (r Custom) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (r Custom) GetExamples(files []string) ([]plan.TestCase, error)
- func (r Custom) GetFiles() ([]string, error)
- func (r Custom) Name() string
- func (r Custom) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type Cypress
- func (c Cypress) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (c Cypress) GetExamples(files []string) ([]plan.TestCase, error)
- func (c Cypress) GetFiles() ([]string, error)
- func (c Cypress) Name() string
- func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type GoTest
- func (g GoTest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (g GoTest) GetExamples(files []string) ([]plan.TestCase, error)
- func (g GoTest) GetFiles() ([]string, error)
- func (g GoTest) Name() string
- func (g GoTest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type GoTestJUnitResult
- type JUnitXMLFailure
- type JUnitXMLSkipped
- type JUnitXMLTestSuite
- type JUnitXMLTestSuites
- type Jest
- func (j Jest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (j Jest) GetExamples(files []string) ([]plan.TestCase, error)
- func (j Jest) GetFiles() ([]string, error)
- func (j Jest) Name() string
- func (j Jest) ParseReport(path string) (JestReport, error)
- func (j Jest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type JestExample
- type JestReport
- type NUnit
- func (n NUnit) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (n NUnit) GetExamples(files []string) ([]plan.TestCase, error)
- func (n NUnit) GetFiles() ([]string, error)
- func (n NUnit) Name() string
- func (n NUnit) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type NUnitJUnitResult
- type Playwright
- func (p Playwright) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)
- func (p Playwright) GetFiles() ([]string, error)
- func (p Playwright) Name() string
- func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type PlaywrightReport
- type PlaywrightReportSuite
- type PlaywrightSpec
- type PlaywrightTest
- type ProcessSignaledError
- type Pytest
- func (p Pytest) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error)
- func (p Pytest) GetFiles() ([]string, error)
- func (p Pytest) Name() string
- func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type PytestPants
- func (p PytestPants) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (p PytestPants) GetExamples(files []string) ([]plan.TestCase, error)
- func (p PytestPants) GetFiles() ([]string, error)
- func (p PytestPants) Name() string
- func (p PytestPants) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type Rspec
- func (r Rspec) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
- func (r Rspec) GetExamples(files []string) ([]plan.TestCase, error)
- func (r Rspec) GetFiles() ([]string, error)
- func (r Rspec) Name() string
- func (r Rspec) ParseReport(path string) (RspecReport, error)
- func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) error
- type RspecExample
- type RspecReport
- type RunResult
- func (r *RunResult) Error() error
- func (r *RunResult) FailedMutedTests() []plan.TestCase
- func (r *RunResult) FailedTests() []plan.TestCase
- func (r *RunResult) MutedTests() []TestResult
- func (r *RunResult) OnlyMutedFailures() bool
- func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)
- func (r *RunResult) SkippedTests() []plan.TestCase
- func (r *RunResult) Statistics() RunStatistics
- func (r *RunResult) Status() RunStatus
- type RunStatistics
- type RunStatus
- type RunnerConfig
- type TestEngineTest
- type TestResult
- type TestRunner
- type TestStatus
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Cucumber ¶
type Cucumber struct {
RunnerConfig
}
func NewCucumber ¶
func NewCucumber(c RunnerConfig) Cucumber
func (Cucumber) CommandNameAndArgs ¶
func (c Cucumber) CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
CommandNameAndArgs replaces placeholders and returns command + args.
func (Cucumber) GetExamples ¶
GetExamples returns an array of test scenarios within the given feature files.
func (Cucumber) GetFiles ¶
GetFiles returns the list of feature files based on include / exclude pattern.
func (Cucumber) ParseReport ¶
func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)
type CucumberDataTableRow ¶
type CucumberDataTableRow struct {
Cells []string `json:"cells"`
}
CucumberDataTableRow represents a row in a step's data table.
type CucumberDocString ¶
type CucumberDocString struct {
Value string `json:"value"`
ContentType string `json:"content_type"`
Line int `json:"line"`
}
CucumberDocString represents a doc string argument for a step.
type CucumberElement ¶
type CucumberElement struct {
ID string `json:"id"`
Keyword string `json:"keyword"`
Name string `json:"name"`
Description string `json:"description"`
Line int `json:"line"`
Type string `json:"type"` // e.g., "scenario", "background"
Steps []CucumberStep `json:"steps"`
Tags []CucumberTag `json:"tags,omitempty"`
}
CucumberElement represents a scenario or background in Cucumber's JSON output.
func (CucumberElement) AggregatedStatus ¶
func (e CucumberElement) AggregatedStatus() string
AggregatedStatus returns overall scenario status based on its steps. It mirrors the logic previously in cucumber.go but uses the parser's structs.
type CucumberFeature ¶
type CucumberFeature struct {
URI string `json:"uri"`
ID string `json:"id"`
Keyword string `json:"keyword"`
Name string `json:"name"`
Description string `json:"description"`
Line int `json:"line"`
Elements []CucumberElement `json:"elements"`
Tags []CucumberTag `json:"tags,omitempty"`
}
CucumberFeature represents a single feature in Cucumber's JSON output.
func ParseCucumberJSONReport ¶
func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error)
ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). This is for actual test results.
type CucumberMatch ¶
type CucumberMatch struct {
Location string `json:"location"` // e.g., "features/step_definitions/steps.rb:5"
}
CucumberMatch represents the matching step definition for a step.
type CucumberResult ¶
type CucumberResult struct {
Status string `json:"status"` // e.g., "passed", "failed", "skipped", "undefined"
ErrorMessage string `json:"error_message,omitempty"`
Duration int64 `json:"duration,omitempty"` // Nanoseconds
}
CucumberResult represents the result of a step execution.
type CucumberStep ¶
type CucumberStep struct {
Keyword string `json:"keyword"`
Name string `json:"name"`
Line int `json:"line"`
Result *CucumberResult `json:"result,omitempty"`
Match *CucumberMatch `json:"match,omitempty"`
DocString *CucumberDocString `json:"doc_string,omitempty"`
DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables
}
CucumberStep represents a single step in a scenario.
type CucumberTag ¶
CucumberTag represents a tag in Cucumber.
type Custom ¶
type Custom struct {
RunnerConfig
}
func NewCustom ¶
func NewCustom(r RunnerConfig) (Custom, error)
func (Custom) CommandNameAndArgs ¶
type Cypress ¶
type Cypress struct {
RunnerConfig
}
func NewCypress ¶
func NewCypress(c RunnerConfig) Cypress
func (Cypress) CommandNameAndArgs ¶
type GoTest ¶
type GoTest struct {
RunnerConfig
}
func NewGoTest ¶
func NewGoTest(c RunnerConfig) GoTest
func (GoTest) CommandNameAndArgs ¶
func (GoTest) GetFiles ¶
GetFiles discovers Go packages using `go list ./...`. Note that "file" does not exist as a first level concept in Golang projects So this func is returning a list of packages instead of files. The implication is that the Server-side smart test splitting will never work. It almost will always fallback to simple splitting.
type GoTestJUnitResult ¶
type GoTestJUnitResult struct {
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Result TestStatus // passed | failed | skipped
Failure *JUnitXMLFailure `xml:"failure"`
Skipped *JUnitXMLSkipped `xml:"skipped"`
}
Struct to decode gotestsun --junitfile=...
type JUnitXMLFailure ¶
type JUnitXMLFailure struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Content string `xml:",chardata"`
}
JUnitXMLFailure represents the <failure> element in JUnit XML
type JUnitXMLSkipped ¶
type JUnitXMLSkipped struct {
Message string `xml:"message,attr"`
}
JUnitXMLSkipped represents the <skipped> element in JUnit XML
type JUnitXMLTestSuite ¶
type JUnitXMLTestSuite struct {
XMLName xml.Name `xml:"testsuite"`
Name string `xml:"name,attr"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Errors int `xml:"errors,attr"`
Time float64 `xml:"time,attr"`
Timestamp string `xml:"timestamp,attr"`
TestCases []GoTestJUnitResult `xml:"testcase"`
}
JUnitXMLTestSuite represents the <testsuite> element in JUnit XML
type JUnitXMLTestSuites ¶
type JUnitXMLTestSuites struct {
XMLName xml.Name `xml:"testsuites"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Errors int `xml:"errors,attr"`
Time float64 `xml:"time,attr"`
TestSuites []JUnitXMLTestSuite `xml:"testsuite"`
}
JUnitXMLTestSuites represents the root <testsuites> element in JUnit XML
type Jest ¶
type Jest struct {
RunnerConfig
}
func NewJest ¶
func NewJest(j RunnerConfig) Jest
func (Jest) CommandNameAndArgs ¶
func (Jest) ParseReport ¶
func (j Jest) ParseReport(path string) (JestReport, error)
type JestExample ¶
type JestReport ¶
type JestReport struct {
NumFailedTests int
TestResults []struct {
AssertionResults []JestExample
FileName string `json:"name"`
Status string
}
}
type NUnit ¶
type NUnit struct {
RunnerConfig
}
func NewNUnit ¶
func NewNUnit(c RunnerConfig) NUnit
func (NUnit) CommandNameAndArgs ¶
type NUnitJUnitResult ¶
type NUnitJUnitResult struct {
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Result TestStatus // passed | failed | skipped
Failure *JUnitXMLFailure `xml:"failure"`
Skipped *JUnitXMLSkipped `xml:"skipped"`
}
NUnitJUnitResult represents a single test case from a JUnit XML report produced by dotnet test --logger junit.
type Playwright ¶
type Playwright struct {
RunnerConfig
}
func NewPlaywright ¶
func NewPlaywright(p RunnerConfig) Playwright
func (Playwright) CommandNameAndArgs ¶
func (Playwright) GetExamples ¶
func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)
GetExamples returns an array of test examples within the given files.
func (Playwright) GetFiles ¶
func (p Playwright) GetFiles() ([]string, error)
func (Playwright) Name ¶
func (p Playwright) Name() string
type PlaywrightReport ¶
type PlaywrightReport struct {
Suites []PlaywrightReportSuite
Stats struct {
Expected int
Unexpected int
}
Errors []struct {
Message string
}
}
type PlaywrightReportSuite ¶
type PlaywrightReportSuite struct {
Title string
Specs []PlaywrightSpec
Suites []PlaywrightReportSuite
}
type PlaywrightSpec ¶
type PlaywrightTest ¶
type ProcessSignaledError ¶
func (*ProcessSignaledError) Error ¶
func (e *ProcessSignaledError) Error() string
type Pytest ¶
type Pytest struct {
RunnerConfig
}
func NewPytest ¶
func NewPytest(c RunnerConfig) Pytest
func (Pytest) CommandNameAndArgs ¶
func (Pytest) GetExamples ¶
GetExamples returns an array of test examples within the given files. It uses `pytest --collect-only -q` to enumerate individual tests.
--tag-filters can be used to filter tests by markers if specified. e.g. --tag-fitlers team:frontend matches markers: with @pytest.mark.execution_tag('team', 'frontend')
The --tag-filters feature also assumes Python Test Collector plugin version >1.2.0 is installed.
type PytestPants ¶
type PytestPants struct {
RunnerConfig
}
func NewPytestPants ¶
func NewPytestPants(c RunnerConfig) PytestPants
func (PytestPants) CommandNameAndArgs ¶
func (PytestPants) GetExamples ¶
func (p PytestPants) GetExamples(files []string) ([]plan.TestCase, error)
func (PytestPants) GetFiles ¶
func (p PytestPants) GetFiles() ([]string, error)
func (PytestPants) Name ¶
func (p PytestPants) Name() string
type Rspec ¶
type Rspec struct {
RunnerConfig
}
func NewRspec ¶
func NewRspec(r RunnerConfig) Rspec
func (Rspec) CommandNameAndArgs ¶
CommandNameAndArgs replaces the "{{testExamples}}" placeholder in the test command with the test cases. It returns the command name and arguments to run the tests.
func (Rspec) GetExamples ¶
GetExamples returns an array of test examples within the given files.
func (Rspec) ParseReport ¶
func (r Rspec) ParseReport(path string) (RspecReport, error)
func (Rspec) Run ¶
Run executes the test command with the given test cases. If retry is true, it will run the command using the retry test command, otherwise it will use the test command.
Error is returned if the command fails to run, exits prematurely, or if the output cannot be parsed.
Test failure is not considered an error, and is instead returned as a RunResult.
type RspecExample ¶
type RspecExample struct {
Id string `json:"id"`
Description string `json:"description"`
FullDescription string `json:"full_description"`
Status string `json:"status"`
FilePath string `json:"file_path"`
LineNumber int `json:"line_number"`
RunTime float64 `json:"run_time"`
}
RspecExample represents a single test example in an Rspec report.
type RspecReport ¶
type RspecReport struct {
Version string `json:"version"`
Seed int `json:"seed"`
Examples []RspecExample `json:"examples"`
Summary struct {
ExampleCount int `json:"example_count"`
FailureCount int `json:"failure_count"`
PendingCount int `json:"pending_count"`
ErrorsOutsideOfExamplesCount int `json:"errors_outside_of_examples_count"`
}
}
RspecReport is the structure for Rspec JSON report.
type RunResult ¶
type RunResult struct {
// contains filtered or unexported fields
}
RunResult is a struct to keep track the results of a test run. It contains the logics to record test results, calculate the status of the run.
func NewRunResult ¶
func (*RunResult) FailedMutedTests ¶
func (*RunResult) FailedTests ¶
FailedTests returns a list of test cases that failed.
func (*RunResult) MutedTests ¶
func (r *RunResult) MutedTests() []TestResult
func (*RunResult) OnlyMutedFailures ¶
func (*RunResult) RecordTestResult ¶
func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)
RecordTestResult records the result of a test case. If the test case found in the mutedTestLookup, it will be marked as muted.
func (*RunResult) SkippedTests ¶
func (*RunResult) Statistics ¶
func (r *RunResult) Statistics() RunStatistics
type RunStatistics ¶
type RunStatus ¶
type RunStatus string
const ( // RunStatusPassed indicates that the run was completed and all tests passed. RunStatusPassed RunStatus = "passed" // RunStatusFailed indicates that the run was completed, but one or more tests failed. RunStatusFailed RunStatus = "failed" // RunStatusError indicates that the run was completed, but there was an error outside of the tests. RunStatusError RunStatus = "error" // RunStatusUnknown indicates that the run status is unknown. // It could be that no tests were run, run was interrupted, or the report is not available. RunStatusUnknown RunStatus = "unknown" )
type RunnerConfig ¶
type RunnerConfig struct {
TestRunner string
// ResultPath is used internally so bktec can read result from Test Runner.
// User typically don't need to worry about setting this except in in RSpec and playwright.
// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
ResultPath string
RetryTestCommand string
TagFilters string
TestCommand string
TestFileExcludePattern string
TestFilePattern string
// contains filtered or unexported fields
}
func (RunnerConfig) LocationPrefix ¶
func (rc RunnerConfig) LocationPrefix() string
type TestEngineTest ¶
type TestEngineTest struct {
Id string
Name string
Scope string
Location string
FileName string `json:"file_name,omitempty"`
Result TestStatus
}
TestEngineTest represents a Test Engine test result object. Some attributes such as `history` and `failure_reason` are omitted as they are not needed by bktec. Ref: https://buildkite.com/docs/test-engine/importing-json#json-test-results-data-reference-test-result-objects
Currently, only pytest and custom runner uses result from test collector.
type TestResult ¶
type TestResult struct {
plan.TestCase
Status TestStatus
ExecutionCount int
Muted bool
}
TestResult is a struct to keep track the result of an individual test case.
type TestRunner ¶
type TestRunner interface {
Run(result *RunResult, testCases []plan.TestCase, retry bool) error
GetExamples(files []string) ([]plan.TestCase, error)
GetFiles() ([]string, error)
Name() string
CommandNameAndArgs(testCases []plan.TestCase, retry bool) (string, []string, error)
LocationPrefix() string
}
func DetectRunner ¶
func DetectRunner(cfg *config.Config) (TestRunner, error)
type TestStatus ¶
type TestStatus string
const ( TestStatusPassed TestStatus = "passed" TestStatusFailed TestStatus = "failed" TestStatusSkipped TestStatus = "skipped" TestStatusUnknown TestStatus = "unknown" )