runner

package
v1.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 26, 2025 License: MIT Imports: 19 Imported by: 0

Documentation

Overview

Package runner provides the test runners that run sets of test cases. Supported runners: rspec, jest, cypress, playwright, pytest, gotest, cucumber.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Cucumber added in v1.6.0

type Cucumber struct {
	RunnerConfig
}

func NewCucumber added in v1.6.0

func NewCucumber(c RunnerConfig) Cucumber

func (Cucumber) GetExamples added in v1.6.0

func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error)

GetExamples returns an array of test scenarios within the given feature files.

func (Cucumber) GetFiles added in v1.6.0

func (c Cucumber) GetFiles() ([]string, error)

GetFiles returns the list of feature files based on include / exclude pattern.

func (Cucumber) Name added in v1.6.0

func (c Cucumber) Name() string

func (Cucumber) ParseReport added in v1.6.0

func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error)

func (Cucumber) Run added in v1.6.0

func (c Cucumber) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes the Cucumber command and records results.

type CucumberDataTableRow added in v1.6.0

type CucumberDataTableRow struct {
	Cells []string `json:"cells"`
}

CucumberDataTableRow represents a row in a step's data table.

type CucumberDocString added in v1.6.0

type CucumberDocString struct {
	Value       string `json:"value"`
	ContentType string `json:"content_type"`
	Line        int    `json:"line"`
}

CucumberDocString represents a doc string argument for a step.

type CucumberElement added in v1.6.0

type CucumberElement struct {
	ID          string         `json:"id"`
	Keyword     string         `json:"keyword"`
	Name        string         `json:"name"`
	Description string         `json:"description"`
	Line        int            `json:"line"`
	Type        string         `json:"type"` // e.g., "scenario", "background"
	Steps       []CucumberStep `json:"steps"`
	Tags        []CucumberTag  `json:"tags,omitempty"`
}

CucumberElement represents a scenario or background in Cucumber's JSON output.

func (CucumberElement) AggregatedStatus added in v1.6.0

func (e CucumberElement) AggregatedStatus() string

AggregatedStatus returns overall scenario status based on its steps. It mirrors the logic previously in cucumber.go but uses the parser's structs.

type CucumberFeature added in v1.6.0

type CucumberFeature struct {
	URI         string            `json:"uri"`
	ID          string            `json:"id"`
	Keyword     string            `json:"keyword"`
	Name        string            `json:"name"`
	Description string            `json:"description"`
	Line        int               `json:"line"`
	Elements    []CucumberElement `json:"elements"`
	Tags        []CucumberTag     `json:"tags,omitempty"`
}

CucumberFeature represents a single feature in Cucumber's JSON output.

func ParseCucumberJSONReport added in v1.6.0

func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error)

ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). This is for actual test results.

type CucumberMatch added in v1.6.0

type CucumberMatch struct {
	Location string `json:"location"` // e.g., "features/step_definitions/steps.rb:5"
}

CucumberMatch represents the matching step definition for a step.

type CucumberResult added in v1.6.0

type CucumberResult struct {
	Status       string `json:"status"` // e.g., "passed", "failed", "skipped", "undefined"
	ErrorMessage string `json:"error_message,omitempty"`
	Duration     int64  `json:"duration,omitempty"` // Nanoseconds
}

CucumberResult represents the result of a step execution.

type CucumberStep added in v1.6.0

type CucumberStep struct {
	Keyword       string                 `json:"keyword"`
	Name          string                 `json:"name"`
	Line          int                    `json:"line"`
	Result        *CucumberResult        `json:"result,omitempty"`
	Match         *CucumberMatch         `json:"match,omitempty"`
	DocString     *CucumberDocString     `json:"doc_string,omitempty"`
	DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables
}

CucumberStep represents a single step in a scenario.

type CucumberTag added in v1.6.0

type CucumberTag struct {
	Name string `json:"name"`
	Line int    `json:"line"`
}

CucumberTag represents a tag in Cucumber.

type Cypress added in v1.1.0

type Cypress struct {
	RunnerConfig
}

func NewCypress added in v1.1.0

func NewCypress(c RunnerConfig) Cypress

func (Cypress) GetExamples added in v1.1.0

func (c Cypress) GetExamples(files []string) ([]plan.TestCase, error)

func (Cypress) GetFiles added in v1.1.0

func (c Cypress) GetFiles() ([]string, error)

func (Cypress) Name added in v1.1.0

func (c Cypress) Name() string

func (Cypress) Run added in v1.1.0

func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

type GoTest added in v1.5.0

type GoTest struct {
	RunnerConfig
}

func NewGoTest added in v1.5.0

func NewGoTest(c RunnerConfig) GoTest

func (GoTest) GetExamples added in v1.5.0

func (g GoTest) GetExamples(files []string) ([]plan.TestCase, error)

func (GoTest) GetFiles added in v1.5.0

func (g GoTest) GetFiles() ([]string, error)

GetFiles discovers Go packages using `go list ./...`. Note that "file" does not exist as a first level concept in Golang projects So this func is returning a list of packages instead of files. The implication is that the Server-side smart test splitting will never work. It almost will always fallback to simple splitting.

func (GoTest) Name added in v1.5.0

func (g GoTest) Name() string

func (GoTest) Run added in v1.5.0

func (g GoTest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes the configured command for the specified packages.

type GoTestJUnitResult added in v1.5.0

type GoTestJUnitResult struct {
	Classname string           `xml:"classname,attr"`
	Name      string           `xml:"name,attr"`
	Result    TestStatus       // passed | failed | skipped
	Failure   *JUnitXMLFailure `xml:"failure"`
	Skipped   *JUnitXMLSkipped `xml:"skipped"`
}

Struct to decode gotestsun --junitfile=...

type JUnitXMLFailure added in v1.5.0

type JUnitXMLFailure struct {
	Message string `xml:"message,attr"`
	Type    string `xml:"type,attr"`
	Content string `xml:",chardata"`
}

JUnitXMLFailure represents the <failure> element in JUnit XML

type JUnitXMLSkipped added in v1.5.0

type JUnitXMLSkipped struct {
	Message string `xml:"message,attr"`
}

JUnitXMLSkipped represents the <skipped> element in JUnit XML

type JUnitXMLTestSuite added in v1.5.0

type JUnitXMLTestSuite struct {
	XMLName   xml.Name            `xml:"testsuite"`
	Name      string              `xml:"name,attr"`
	Tests     int                 `xml:"tests,attr"`
	Failures  int                 `xml:"failures,attr"`
	Errors    int                 `xml:"errors,attr"`
	Time      float64             `xml:"time,attr"`
	Timestamp string              `xml:"timestamp,attr"`
	TestCases []GoTestJUnitResult `xml:"testcase"`
}

JUnitXMLTestSuite represents the <testsuite> element in JUnit XML

type JUnitXMLTestSuites added in v1.5.0

type JUnitXMLTestSuites struct {
	XMLName    xml.Name            `xml:"testsuites"`
	Tests      int                 `xml:"tests,attr"`
	Failures   int                 `xml:"failures,attr"`
	Errors     int                 `xml:"errors,attr"`
	Time       float64             `xml:"time,attr"`
	TestSuites []JUnitXMLTestSuite `xml:"testsuite"`
}

JUnitXMLTestSuites represents the root <testsuites> element in JUnit XML

type Jest

type Jest struct {
	RunnerConfig
}

func NewJest

func NewJest(j RunnerConfig) Jest

func (Jest) GetExamples

func (j Jest) GetExamples(files []string) ([]plan.TestCase, error)

func (Jest) GetFiles

func (j Jest) GetFiles() ([]string, error)

GetFiles returns an array of file names using the discovery pattern.

func (Jest) Name

func (j Jest) Name() string

func (Jest) ParseReport

func (j Jest) ParseReport(path string) (JestReport, error)

func (Jest) Run

func (j Jest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

type JestExample

type JestExample struct {
	Name           string   `json:"fullName"`
	Status         string   `json:"status"`
	Title          string   `json:"title"`
	AncestorTitles []string `json:"ancestorTitles"`
	Location       struct {
		Line   int
		Column int
	}
}

type JestReport

type JestReport struct {
	NumFailedTests            int
	NumRuntimeErrorTestSuites int
	TestResults               []struct {
		AssertionResults []JestExample
		FileName         string `json:"name"`
	}
}

type Playwright added in v1.1.0

type Playwright struct {
	RunnerConfig
}

func NewPlaywright added in v1.1.0

func NewPlaywright(p RunnerConfig) Playwright

func (Playwright) GetExamples added in v1.1.0

func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error)

func (Playwright) GetFiles added in v1.1.0

func (p Playwright) GetFiles() ([]string, error)

func (Playwright) Name added in v1.1.0

func (p Playwright) Name() string

func (Playwright) Run added in v1.1.0

func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

type PlaywrightReport added in v1.1.0

type PlaywrightReport struct {
	Suites []PlaywrightReportSuite
	Stats  struct {
		Expected   int
		Unexpected int
	}
	Errors []struct {
		Message string
	}
}

type PlaywrightReportSuite added in v1.1.0

type PlaywrightReportSuite struct {
	Title  string
	Specs  []PlaywrightSpec
	Suites []PlaywrightReportSuite
}

type PlaywrightSpec added in v1.1.0

type PlaywrightSpec struct {
	File   string
	Line   int
	Column int
	Id     string
	Title  string
	Ok     bool
	Tests  []PlaywrightTest
}

type PlaywrightTest added in v1.2.0

type PlaywrightTest struct {
	ProjectName string
	Status      string
}

type ProcessSignaledError

type ProcessSignaledError struct {
	Signal syscall.Signal
}

func (*ProcessSignaledError) Error

func (e *ProcessSignaledError) Error() string

type Pytest added in v1.4.0

type Pytest struct {
	RunnerConfig
}

func NewPytest added in v1.4.0

func NewPytest(c RunnerConfig) Pytest

func (Pytest) GetExamples added in v1.4.0

func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error)

func (Pytest) GetFiles added in v1.4.0

func (p Pytest) GetFiles() ([]string, error)

func (Pytest) Name added in v1.4.0

func (p Pytest) Name() string

func (Pytest) Run added in v1.4.0

func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

type PytestPants added in v1.6.0

type PytestPants struct {
	RunnerConfig
}

func NewPytestPants added in v1.6.0

func NewPytestPants(c RunnerConfig) PytestPants

func (PytestPants) GetExamples added in v1.6.0

func (p PytestPants) GetExamples(files []string) ([]plan.TestCase, error)

func (PytestPants) GetFiles added in v1.6.0

func (p PytestPants) GetFiles() ([]string, error)

func (PytestPants) Name added in v1.6.0

func (p PytestPants) Name() string

func (PytestPants) Run added in v1.6.0

func (p PytestPants) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

type Rspec

type Rspec struct {
	RunnerConfig
}

func NewRspec

func NewRspec(r RunnerConfig) Rspec

func (Rspec) GetExamples

func (r Rspec) GetExamples(files []string) ([]plan.TestCase, error)

GetExamples returns an array of test examples within the given files.

func (Rspec) GetFiles

func (r Rspec) GetFiles() ([]string, error)

GetFiles returns an array of file names using the discovery pattern.

func (Rspec) Name

func (r Rspec) Name() string

func (Rspec) ParseReport

func (r Rspec) ParseReport(path string) (RspecReport, error)

func (Rspec) Run

func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) error

Run executes the test command with the given test cases. If retry is true, it will run the command using the retry test command, otherwise it will use the test command.

Error is returned if the command fails to run, exits prematurely, or if the output cannot be parsed.

Test failure is not considered an error, and is instead returned as a RunResult.

type RspecExample

type RspecExample struct {
	Id              string  `json:"id"`
	Description     string  `json:"description"`
	FullDescription string  `json:"full_description"`
	Status          string  `json:"status"`
	FilePath        string  `json:"file_path"`
	LineNumber      int     `json:"line_number"`
	RunTime         float64 `json:"run_time"`
}

RspecExample represents a single test example in an Rspec report.

type RspecReport

type RspecReport struct {
	Version  string         `json:"version"`
	Seed     int            `json:"seed"`
	Examples []RspecExample `json:"examples"`
	Summary  struct {
		ExampleCount                 int `json:"example_count"`
		FailureCount                 int `json:"failure_count"`
		PendingCount                 int `json:"pending_count"`
		ErrorsOutsideOfExamplesCount int `json:"errors_outside_of_examples_count"`
	}
}

RspecReport is the structure for Rspec JSON report.

type RunResult

type RunResult struct {
	// contains filtered or unexported fields
}

RunResult is a struct to keep track the results of a test run. It contains the logics to record test results, calculate the status of the run.

func NewRunResult added in v1.2.0

func NewRunResult(mutedTests []plan.TestCase) *RunResult

func (*RunResult) Error added in v1.2.1

func (r *RunResult) Error() error

func (*RunResult) FailedMutedTests added in v1.5.0

func (r *RunResult) FailedMutedTests() []plan.TestCase

func (*RunResult) FailedTests

func (r *RunResult) FailedTests() []plan.TestCase

FailedTests returns a list of test cases that failed.

func (*RunResult) MutedTests added in v1.2.0

func (r *RunResult) MutedTests() []TestResult

func (*RunResult) RecordTestResult added in v1.2.0

func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus)

RecordTestResult records the result of a test case. If the test case found in the mutedTestLookup, it will be marked as muted.

func (*RunResult) SkippedTests added in v1.3.0

func (r *RunResult) SkippedTests() []plan.TestCase

func (*RunResult) Statistics added in v1.2.0

func (r *RunResult) Statistics() RunStatistics

func (*RunResult) Status

func (r *RunResult) Status() RunStatus

Status returns the overall status of the test run. If there is an error, it returns RunStatusError. If there are failed tests, it returns RunStatusFailed. Otherwise, it returns RunStatusPassed.

type RunStatistics added in v1.2.0

type RunStatistics struct {
	Total            int `json:"total"`
	PassedOnFirstRun int `json:"passed_on_first_run"`
	PassedOnRetry    int `json:"passed_on_retry"`
	MutedPassed      int `json:"muted_passed"`
	MutedFailed      int `json:"muted_failed"`
	Failed           int `json:"failed"`
	Skipped          int `json:"skipped"`
}

type RunStatus

type RunStatus string
const (
	// RunStatusPassed indicates that the run was completed and all tests passed.
	RunStatusPassed RunStatus = "passed"
	// RunStatusFailed indicates that the run was completed, but one or more tests failed.
	RunStatusFailed RunStatus = "failed"
	// RunStatusError indicates that the run was completed, but there was an error outside of the tests.
	RunStatusError RunStatus = "error"
	// RunStatusUnknown indicates that the run status is unknown.
	// It could be that no tests were run, run was interrupted, or the report is not available.
	RunStatusUnknown RunStatus = "unknown"
)

type RunnerConfig

type RunnerConfig struct {
	TestRunner             string
	TestCommand            string
	TestFilePattern        string
	TestFileExcludePattern string
	RetryTestCommand       string
	// ResultPath is used internally so bktec can read result from Test Runner.
	// User typically don't need to worry about setting this except in in RSpec and playwright.
	// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
	ResultPath string
}

type TestEngineTest added in v1.4.0

type TestEngineTest struct {
	Id       string
	Name     string
	Scope    string
	Location string
	FileName string `json:"file_name,omitempty"`
	Result   TestStatus
}

TestEngineTest represents a Test Engine test result object. Some attributes such as `history` and `failure_reason` are omitted as they are not needed by bktec. Ref: https://buildkite.com/docs/test-engine/importing-json#json-test-results-data-reference-test-result-objects

Currently, only pytest uses result from test collector. If we use this somewhere else in the future, we may want to extract this struct.

func ParsePytestCollectorResult added in v1.4.0

func ParsePytestCollectorResult(path string) ([]TestEngineTest, error)

type TestResult added in v1.2.0

type TestResult struct {
	plan.TestCase
	Status         TestStatus
	ExecutionCount int
	Muted          bool
}

TestResult is a struct to keep track the result of an individual test case.

type TestRunner

type TestRunner interface {
	Run(result *RunResult, testCases []plan.TestCase, retry bool) error
	GetExamples(files []string) ([]plan.TestCase, error)
	GetFiles() ([]string, error)
	Name() string
}

func DetectRunner

func DetectRunner(cfg config.Config) (TestRunner, error)

type TestStatus added in v1.2.0

type TestStatus string
const (
	TestStatusPassed  TestStatus = "passed"
	TestStatusFailed  TestStatus = "failed"
	TestStatusSkipped TestStatus = "skipped"
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL