Documentation
¶
Overview ¶
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Index ¶
- Constants
- func Build(id string, ctx cruntime.ReconcileRequestContext) (base.Engine, error)
- type Affinity
- type Alluxio
- type AlluxioEngine
- func (e *AlluxioEngine) AssignNodesToCache(desiredNum int32) (currentScheduleNum int32, err error)
- func (e *AlluxioEngine) BindToDataset() (err error)
- func (e *AlluxioEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)
- func (e *AlluxioEngine) CheckMasterReady() (ready bool, err error)
- func (e *AlluxioEngine) CheckRuntimeHealthy() (err error)
- func (e *AlluxioEngine) CheckWorkersReady() (ready bool, err error)
- func (e *AlluxioEngine) CreateVolume() (err error)
- func (e *AlluxioEngine) DeleteVolume() (err error)
- func (e *AlluxioEngine) FreeStorageBytes() (value int64, err error)
- func (e *AlluxioEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)
- func (e *AlluxioEngine) GetMetadataFileName() string
- func (e *AlluxioEngine) GetMetadataInfoFile() string
- func (e *AlluxioEngine) GetMetadataInfoFileName() string
- func (e *AlluxioEngine) GetWorkerUsedCapacity() (map[string]int64, error)
- func (e *AlluxioEngine) PrepareUFS() (err error)
- func (e *AlluxioEngine) RestoreMetadataInternal() (err error)
- func (e *AlluxioEngine) SetupMaster() (err error)
- func (e *AlluxioEngine) SetupWorkers() (err error)
- func (e *AlluxioEngine) ShouldCheckUFS() (should bool, err error)
- func (e *AlluxioEngine) ShouldSetupMaster() (should bool, err error)
- func (e *AlluxioEngine) ShouldSetupWorkers() (should bool, err error)
- func (e *AlluxioEngine) Shutdown() (err error)
- func (e *AlluxioEngine) SyncMetadata() (err error)
- func (e *AlluxioEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)
- func (e *AlluxioEngine) TotalFileNums() (value int64, err error)
- func (e *AlluxioEngine) TotalStorageBytes() (value int64, err error)
- func (e *AlluxioEngine) UpdateCacheOfDataset() (err error)
- func (e *AlluxioEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)
- func (e *AlluxioEngine) UsedStorageBytes() (value int64, err error)
- type Fuse
- type HadoopConfig
- type ImageInfo
- type InitUsers
- type JobMaster
- type JobWorker
- type Journal
- type Level
- type Master
- type MetadataSyncResult
- type Metastore
- type NodeAffinity
- type NodeSelector
- type NodeSelectorRequirement
- type NodeSelectorTerm
- type Ports
- type Restore
- type ShortCircuit
- type Tieredstore
- type UFSPath
- type UFSVolume
- type UserInfo
- type Worker
Constants ¶
const ( // alluxioUser string = "fluid" METRICS_PREFIX_BYTES_READ_LOCAL = "Cluster.BytesReadLocal " METRICS_PREFIX_BYTES_READ_REMOTE = "Cluster.BytesReadRemote " METRICS_PREFIX_BYTES_READ_UFS_ALL = "Cluster.BytesReadUfsAll " METRICS_PREFIX_BYTES_READ_LOCAL_THROUGHPUT = "Cluster.BytesReadLocalThroughput " METRICS_PREFIX_BYTES_READ_REMOTE_THROUGHPUT = "Cluster.BytesReadRemoteThroughput " METRICS_PREFIX_BYTES_READ_UFS_THROUGHPUT = "Cluster.BytesReadUfsThroughput " SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Capacity: " SUMMARY_PREFIX_USED_CAPACITY = "Used Capacity: " METADATA_SYNC_NOT_DONE_MSG = "[Calculating]" ALLUXIO_RUNTIME_METRICS_LABEL = "alluxio_runtime_metrics" CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500 AUTO_SELECT_PORT_MIN = 20000 AUTO_SELECT_PORT_MAX = 30000 PORT_NUM = 9 CACHE_HIT_QUERY_INTERVAL_MIN = 1 HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml" HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml" HADOOP_CONF_MOUNT_PATH = "/hdfs-config" )
Variables ¶
This section is empty.
Functions ¶
Types ¶
type Affinity ¶ added in v0.3.0
type Affinity struct {
NodeAffinity *NodeAffinity `yaml:"nodeAffinity"`
}
type Alluxio ¶
type Alluxio struct {
FullnameOverride string `yaml:"fullnameOverride"`
ImageInfo `yaml:",inline"`
UserInfo `yaml:",inline"`
NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
JvmOptions []string `yaml:"jvmOptions,omitempty"`
Properties map[string]string `yaml:"properties,omitempty"`
Master Master `yaml:"master,omitempty"`
JobMaster JobMaster `yaml:"jobMaster,omitempty"`
Worker Worker `yaml:"worker,omitempty"`
JobWorker JobWorker `yaml:"jobWorker,omitempty"`
Fuse Fuse `yaml:"fuse,omitempty"`
Tieredstore Tieredstore `yaml:"tieredstore,omitempty"`
Metastore Metastore `yaml:"metastore,omitempty"`
Journal Journal `yaml:"journal,omitempty"`
ShortCircuit ShortCircuit `yaml:"shortCircuit,omitempty"`
UFSPaths []UFSPath `yaml:"ufsPaths,omitempty"`
UFSVolumes []UFSVolume `yaml:"ufsVolumes,omitempty"`
InitUsers InitUsers `yaml:"initUsers,omitempty"`
Monitoring string `yaml:"monitoring,omitempty"`
HadoopConfig HadoopConfig `yaml:"hadoopConfig,omitempty"`
Tolerations []v1.Toleration `yaml:"tolerations,omitempty"`
}
The value yaml file
type AlluxioEngine ¶
type AlluxioEngine struct {
Log logr.Logger
client.Client
MetadataSyncDoneCh chan MetadataSyncResult
UnitTest bool
// contains filtered or unexported fields
}
AlluxioEngine implements the Engine interface.
func (*AlluxioEngine) AssignNodesToCache ¶
func (e *AlluxioEngine) AssignNodesToCache(desiredNum int32) (currentScheduleNum int32, err error)
AssignNodesToCache finds nodes to place the cache engine
func (*AlluxioEngine) BindToDataset ¶ added in v0.5.0
func (e *AlluxioEngine) BindToDataset() (err error)
func (*AlluxioEngine) CheckAndUpdateRuntimeStatus ¶
func (e *AlluxioEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)
CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.
func (*AlluxioEngine) CheckMasterReady ¶
func (e *AlluxioEngine) CheckMasterReady() (ready bool, err error)
CheckMasterReady checks if the master is ready
func (*AlluxioEngine) CheckRuntimeHealthy ¶
func (e *AlluxioEngine) CheckRuntimeHealthy() (err error)
CheckRuntimeHealthy checks the healthy of the runtime
func (*AlluxioEngine) CheckWorkersReady ¶
func (e *AlluxioEngine) CheckWorkersReady() (ready bool, err error)
are the workers ready
func (*AlluxioEngine) CreateVolume ¶
func (e *AlluxioEngine) CreateVolume() (err error)
CreateVolume creates volume
func (*AlluxioEngine) DeleteVolume ¶
func (e *AlluxioEngine) DeleteVolume() (err error)
DeleteVolume creates volume
func (*AlluxioEngine) FreeStorageBytes ¶
func (e *AlluxioEngine) FreeStorageBytes() (value int64, err error)
FreeStorageBytes returns free storage size of Alluxio in bytes
func (*AlluxioEngine) GetHCFSStatus ¶ added in v0.4.0
func (e *AlluxioEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)
Query the hcfs status
func (*AlluxioEngine) GetMetadataFileName ¶ added in v0.5.0
func (e *AlluxioEngine) GetMetadataFileName() string
func (*AlluxioEngine) GetMetadataInfoFile ¶ added in v0.5.0
func (e *AlluxioEngine) GetMetadataInfoFile() string
func (*AlluxioEngine) GetMetadataInfoFileName ¶ added in v0.5.0
func (e *AlluxioEngine) GetMetadataInfoFileName() string
func (*AlluxioEngine) GetWorkerUsedCapacity ¶ added in v0.5.0
func (e *AlluxioEngine) GetWorkerUsedCapacity() (map[string]int64, error)
GetWorkerUsedCapacity gets cache capacity usage for each worker as a map. It parses result from stdout when executing `alluxio fsadmin report capacity` command and extracts worker name(IP or hostname) along with used capacity for that worker
func (*AlluxioEngine) PrepareUFS ¶
func (e *AlluxioEngine) PrepareUFS() (err error)
PrepareUFS do all the UFS preparations
func (*AlluxioEngine) RestoreMetadataInternal ¶ added in v0.5.0
func (e *AlluxioEngine) RestoreMetadataInternal() (err error)
RestoreMetadataInternal restore metadata from backup there are three kinds of data to be restored 1. metadata of dataset 2. ufsTotal info of dataset 3. fileNum info of dataset if 1 fails, the alluxio master will fail directly, if 2 or 3 fails, fluid will get the info from alluxio again
func (*AlluxioEngine) SetupMaster ¶
func (e *AlluxioEngine) SetupMaster() (err error)
SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered
func (*AlluxioEngine) SetupWorkers ¶
func (e *AlluxioEngine) SetupWorkers() (err error)
SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.
func (*AlluxioEngine) ShouldCheckUFS ¶
func (e *AlluxioEngine) ShouldCheckUFS() (should bool, err error)
ShouldCheckUFS checks if it requires checking UFS
func (*AlluxioEngine) ShouldSetupMaster ¶
func (e *AlluxioEngine) ShouldSetupMaster() (should bool, err error)
ShouldSetupMaster checks if we need setup the master
func (*AlluxioEngine) ShouldSetupWorkers ¶
func (e *AlluxioEngine) ShouldSetupWorkers() (should bool, err error)
ShouldSetupWorkers checks if we need setup the workers
func (*AlluxioEngine) Shutdown ¶
func (e *AlluxioEngine) Shutdown() (err error)
shut down the Alluxio engine
func (*AlluxioEngine) SyncMetadata ¶ added in v0.4.0
func (e *AlluxioEngine) SyncMetadata() (err error)
SyncMetadata syncs metadata if necessary For Alluxio Engine, metadata sync is an asynchronous operation, which means you should call this function periodically to make sure the function actually takes effect.
func (*AlluxioEngine) SyncReplicas ¶
func (e *AlluxioEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)
SyncReplicas syncs the replicas
func (*AlluxioEngine) TotalFileNums ¶
func (e *AlluxioEngine) TotalFileNums() (value int64, err error)
TotalFileNums returns the total num of files in Alluxio
func (*AlluxioEngine) TotalStorageBytes ¶
func (e *AlluxioEngine) TotalStorageBytes() (value int64, err error)
return total storage size of Alluxio in bytes
func (*AlluxioEngine) UpdateCacheOfDataset ¶
func (e *AlluxioEngine) UpdateCacheOfDataset() (err error)
func (*AlluxioEngine) UpdateDatasetStatus ¶
func (e *AlluxioEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)
Bind to the dataset
func (*AlluxioEngine) UsedStorageBytes ¶
func (e *AlluxioEngine) UsedStorageBytes() (value int64, err error)
UsedStorageBytes returns used storage size of Alluxio in bytes
type Fuse ¶
type Fuse struct {
Image string `yaml:"image,omitempty"`
NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
ImageTag string `yaml:"imageTag,omitempty"`
ImagePullPolicy string `yaml:"imagePullPolicy,omitempty"`
Properties map[string]string `yaml:"properties,omitempty"`
Env map[string]string `yaml:"env,omitempty"`
JvmOptions []string `yaml:"jvmOptions,omitempty"`
MountPath string `yaml:"mountPath,omitempty"`
ShortCircuitPolicy string `yaml:"shortCircuitPolicy,omitempty"`
Args []string `yaml:"args,omitempty"`
HostNetwork bool `yaml:"hostNetwork,omitempty"`
Enabled bool `yaml:"enabled,omitempty"`
Resources common.Resources `yaml:"resources,omitempty"`
Global bool `yaml:"global,omitempty"`
}
type HadoopConfig ¶ added in v0.5.0
type InitUsers ¶ added in v0.3.0
type InitUsers struct {
ImageInfo `yaml:",inline"`
//Args []string `yaml:"args"`
EnvUsers string `yaml:"envUsers"`
Dir string `yaml:"dir"`
Enabled bool `yaml:"enabled,omitempty"`
EnvTieredPaths string `yaml:"envTieredPaths"`
}
The container to init the users
type Level ¶
type Level struct {
Alias string `yaml:"alias,omitempty"`
Level int `yaml:"level"`
Mediumtype string `yaml:"mediumtype,omitempty"`
Type string `yaml:"type,omitempty"`
Path string `yaml:"path,omitempty"`
Quota string `yaml:"quota,omitempty"`
High string `yaml:"high,omitempty"`
Low string `yaml:"low,omitempty"`
}
type Master ¶
type Master struct {
JvmOptions []string `yaml:"jvmOptions,omitempty"`
Env map[string]string `yaml:"env,omitempty"`
Affinity Affinity `yaml:"affinity"`
NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
Properties map[string]string `yaml:"properties,omitempty"`
Replicas int32 `yaml:"replicaCount,omitempty"`
HostNetwork bool `yaml:"hostNetwork,omitempty"`
Resources common.Resources `yaml:"resources,omitempty"`
Ports Ports `yaml:"ports,omitempty"`
BackupPath string `yaml:"backupPath,omitempty"`
Restore Restore `yaml:"restore,omitempty"`
}
type MetadataSyncResult ¶ added in v0.4.0
type MetadataSyncResult struct {
Done bool
StartTime time.Time
UfsTotal string
FileNum string
Err error
}
MetadataSyncResult describes result for asynchronous metadata sync
type NodeAffinity ¶ added in v0.3.0
type NodeAffinity struct {
// If the affinity requirements specified by this field are not met at
// scheduling time, the pod will not be scheduled onto the node.
// If the affinity requirements specified by this field cease to be met
// at some point during pod execution (e.g. due to an update), the system
// may or may not try to eventually evict the pod from its node.
// +optional
RequiredDuringSchedulingIgnoredDuringExecution *NodeSelector `yaml:"requiredDuringSchedulingIgnoredDuringExecution"`
}
type NodeSelector ¶ added in v0.3.0
type NodeSelector struct {
//Required. A list of node selector terms.
NodeSelectorTerms []NodeSelectorTerm `yaml:"nodeSelectorTerms"`
}
NodeSelector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.
type NodeSelectorRequirement ¶ added in v0.3.0
type NodeSelectorRequirement struct {
// The label key that the selector applies to.
Key string `yaml:"key,omitempty"`
// Represents a key's relationship to a set of values.
// Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
Operator string `yaml:"operator,omitempty"`
// +optional
Values []string `yaml:"values,omitempty"`
}
NodeSelectorRequirement is a selector that contains values, a key, and an operator that relates the key and values.
type NodeSelectorTerm ¶ added in v0.3.0
type NodeSelectorTerm struct {
// A list of node selector requirements by node's labels.
MatchExpressions []NodeSelectorRequirement `yaml:"matchExpressions"`
}
NodeSelectorTerm represents expressions and fields required to select nodes. A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
type ShortCircuit ¶
type Tieredstore ¶
type Tieredstore struct {
Levels []Level `yaml:"levels,omitempty"`
}
type Worker ¶
type Worker struct {
JvmOptions []string `yaml:"jvmOptions,omitempty"`
Env map[string]string `yaml:"env,omitempty"`
NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
Properties map[string]string `yaml:"properties,omitempty"`
HostNetwork bool `yaml:"hostNetwork,omitempty"`
Resources common.Resources `yaml:"resources,omitempty"`
Ports Ports `yaml:"ports,omitempty"`
}
Source Files
¶
- cache.go
- const.go
- create_volume.go
- dataset.go
- delete_volume.go
- engine.go
- hcfs.go
- health_check.go
- label.go
- master.go
- master_internal.go
- metadata.go
- node.go
- replicas.go
- runtime_info.go
- shutdown.go
- status.go
- transform.go
- transform_fuse.go
- transform_hadoop_config.go
- transform_init_users.go
- transform_optimization.go
- transform_permission.go
- transform_resources.go
- transform_ufs.go
- types.go
- types_selector.go
- ufs.go
- ufs_internal.go
- utils.go
- worker.go