experiment

package
v0.1.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 3, 2021 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Overview

Package experiment enables extraction of useful information from experiment objects and their formatting.

Index

Examples

Constants

This section is empty.

Variables

View Source 
var GetClient = func() (rc client.Client, err error) {
	var restConf *rest.Config
	restConf, err = GetConfig()
	if err != nil {
		return nil, err
	}

	var addKnownTypes = func(scheme *runtime.Scheme) error {

		metav1.AddToGroupVersion(scheme, v2alpha2.GroupVersion)
		scheme.AddKnownTypes(v2alpha2.GroupVersion, &v2alpha2.Experiment{})
		scheme.AddKnownTypes(v2alpha2.GroupVersion, &v2alpha2.ExperimentList{})
		return nil
	}

	var schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	scheme := runtime.NewScheme()
	err = schemeBuilder.AddToScheme(scheme)

	if err == nil {
		rc, err = client.New(restConf, client.Options{
			Scheme: scheme,
		})
		if err == nil {
			return rc, nil
		}
	}
	return nil, errors.New("cannot get client using rest config")
}

GetClient constructs and returns a K8s client. The returned client has experiment types registered.

View Source 
var GetConfig = func() (*rest.Config, error) {
	return config.GetConfig()
}

GetConfig variable is useful for test mocks.

Functions

func GetMetricNameAndUnits 

func GetMetricNameAndUnits(metricInfo v2alpha2.MetricInfo) string

GetMetricNameAndUnits extracts the name, and if specified, units for the given metricInfo object and combines them into a string.

Example 
u := "inches"
mi := v2alpha2.MetricInfo{
	Name: "height",
	MetricObj: v2alpha2.Metric{
		Spec: v2alpha2.MetricSpec{
			Units: &u,
		},
	},
}
met := GetMetricNameAndUnits(mi)
fmt.Println(met)

Output:

height (inches)
Example (Unitless) 
mi := v2alpha2.MetricInfo{
	Name:      "weight",
	MetricObj: v2alpha2.Metric{},
}
met := GetMetricNameAndUnits(mi)
fmt.Println(met)

Output:

weight

func StringifyObjective 

func StringifyObjective(objective v2alpha2.Objective) string

StringifyObjective returns a string representation of the given objective.

Example (Lowerlimit) 
q := resource.MustParse("0.998")
obj := v2alpha2.Objective{
	Metric:     "accuracy",
	UpperLimit: nil,
	LowerLimit: &q,
}
str := StringifyObjective(obj)
fmt.Println(str)

Output:

0.998 <= accuracy
Example (Upperandlower) 
q1 := resource.MustParse("6.998")
q2 := resource.MustParse("7.012")
obj := v2alpha2.Objective{
	Metric:     "pH level",
	UpperLimit: &q2,
	LowerLimit: &q1,
}
str := StringifyObjective(obj)
fmt.Println(str)

Output:

6.998 <= pH level <= 7.012
Example (Upperlimit) 
q := resource.MustParse("0.01")
obj := v2alpha2.Objective{
	Metric:     "error-rate",
	UpperLimit: &q,
	LowerLimit: nil,
}
str := StringifyObjective(obj)
fmt.Println(str)

Output:

error-rate <= 0.010

func StringifyReward added in v0.1.4 

func StringifyReward(reward v2alpha2.Reward) string

StringifyReward returns a string representation of the given reward.

Types

type ConditionType added in v0.1.5 

type ConditionType string

ConditionType is a type for conditions that can be asserted 

const (
	// Completed implies experiment is complete
	Completed ConditionType = "completed"

	// WinnerFound implies experiment has found a winner
	WinnerFound ConditionType = "winnerFound"
)

type Experiment 

type Experiment struct {
	v2alpha2.Experiment
}

Experiment is an enhancement of v2alpha2.Experiment struct, and supports various methods used in describing an experiment.

func GetExperiment added in v0.1.5 

func GetExperiment(latest bool, name string, namespace string) (*Experiment, error)

GetExperiment gets the experiment from cluster

func (*Experiment) Assert added in v0.1.5 

func (e *Experiment) Assert(conditions []ConditionType) error

Assert verifies a given set of conditions for the experiment.

func (*Experiment) Completed added in v0.1.5 

func (e *Experiment) Completed() bool

Completed indicates if the experiment has completed.

func (*Experiment) GetAnnotatedMetricStrs added in v0.1.4 

func (e *Experiment) GetAnnotatedMetricStrs(reward v2alpha2.Reward) []string

GetAnnotatedMetricStrs returns a slice of values for a reward

func (*Experiment) GetMetricDec added in v0.1.4 

func (e *Experiment) GetMetricDec(metric string, version string) *inf.Dec

GetMetricDec returns the metric value as a string for a given metric and a given version.

func (*Experiment) GetMetricStr 

func (e *Experiment) GetMetricStr(metric string, version string) string

GetMetricStr returns the metric value as a string for a given metric and a given version.

Example 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 'mean-latency' metric for 'canary' version
met := exp.GetMetricStr("mean-latency", "canary")
fmt.Println(met)

Output:

197.500
Example (Unavailable1) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 'fake' metric for 'default' version
met := exp.GetMetricStr("fake", "default")
fmt.Println(met)

Output:

unavailable
Example (Unavailable2) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 'mean-latency' metric for 'perfect' version
met := exp.GetMetricStr("mean-latency", "perfect")
fmt.Println(met)

Output:

unavailable

func (*Experiment) GetMetricStrs 

func (e *Experiment) GetMetricStrs(metric string) []string

GetMetricStrs returns the given metric's value as a slice of strings, whose elements correspond to versions.

Example 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 'mean-latency' metric for all versions ('default' and 'canary')
mets := exp.GetMetricStrs("mean-latency")
fmt.Println(mets)

Output:

[228.788 197.500]
Example (Unavailable) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 'fake' metric for all versions ('default' and 'canary')
mets := exp.GetMetricStrs("fake")
fmt.Println(mets)

Output:

[unavailable unavailable]

func (*Experiment) GetSatisfyStr 

func (e *Experiment) GetSatisfyStr(objectiveIndex int, version string) string

GetSatisfyStr returns a true/false/unavailable valued string denotating if a version satisfies the objective.

Example 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 2nd objective for 'canary' version
obj := exp.GetSatisfyStr(1, "canary")
fmt.Println(obj)

Output:

true
Example (Unavailable1) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 3rd objective for 'default' version
// This experiment has only two objectives, so this value is unavailable
obj := exp.GetSatisfyStr(2, "default")
fmt.Println(obj)

Output:

unavailable
Example (Unavailable2) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment2.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of the 2nd objective for 'perfect' version
obj := exp.GetSatisfyStr(1, "perfect")
fmt.Println(obj)

Output:

unavailable

func (*Experiment) GetSatisfyStrs 

func (e *Experiment) GetSatisfyStrs(objectiveIndex int) []string

GetSatisfyStrs returns a slice of true/false/unavailable valued strings for an objective denoting if it is satisfied by versions.

Example 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of objective indicators for the 2nd objective for all versions ('default' and 'canary')
objs := exp.GetSatisfyStrs(1)
fmt.Println(objs)

Output:

[true true]
Example (Unavailable1) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment2.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of objective indicators for the 2nd objective for all versions ('default' and 'canary')
// This experiment does not have versionInfo as part of its spec section, so there are no versions
objs := exp.GetSatisfyStrs(1)
fmt.Println(objs)

Output:

[]
Example (Unavailable2) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of objective indicators for the 3nd objective for all versions ('default' and 'canary')
// The experiment has only two objectives, so these values are unavailable.
objs := exp.GetSatisfyStrs(2)
fmt.Println(objs)

Output:

[unavailable unavailable]

func (*Experiment) GetVersions 

func (e *Experiment) GetVersions() []string

GetVersions returns the slice of version name strings. If the VersionInfo section is not present in the experiment's spec, then this slice is empty.

Example 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of objective indicators for the 2nd objective for all versions ('default' and 'canary')
versions := exp.GetVersions()
fmt.Println(versions)

Output:

[default canary]
Example (Empty) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment2.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
// Get value of objective indicators for the 2nd objective for all versions ('default' and 'canary')
// This experiment does not have versionInfo as part of its spec section, so there are no versions
versions := exp.GetVersions()
fmt.Println(versions)

Output:

[]

func (*Experiment) Started 

func (e *Experiment) Started() bool

Started indicates if at least one iteration of the experiment has completed.

Example (False) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment2.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
started := exp.Started()
fmt.Println(started)

Output:

false
Example (True) 
// Read in an experiment object from the testdata folder
filePath := utils.CompletePath("../testdata", "experiment3.yaml")
buf, _ := ioutil.ReadFile(filePath)
exp := &Experiment{}
yaml.Unmarshal(buf, exp)
started := exp.Started()
fmt.Println(started)

Output:

true

func (*Experiment) WinnerFound added in v0.1.5 

func (e *Experiment) WinnerFound() bool

WinnerFound indicates if the experiment has found a winning version (winner).

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.