Documentation
¶
Index ¶
- type AggregationFunction
- type AggregationFunction_Operation_ApproxCountDistinct
- type AggregationFunction_Operation_ApproxPercentile
- type AggregationFunction_Operation_Avg
- type AggregationFunction_Operation_CountFunction
- type AggregationFunction_Operation_First
- type AggregationFunction_Operation_FirstDistinct
- type AggregationFunction_Operation_FirstN
- type AggregationFunction_Operation_Last
- type AggregationFunction_Operation_LastDistinct
- type AggregationFunction_Operation_LastN
- type AggregationFunction_Operation_Max
- type AggregationFunction_Operation_Min
- type AggregationFunction_Operation_StddevPop
- type AggregationFunction_Operation_StddevSamp
- type AggregationFunction_Operation_Sum
- type AggregationFunction_Operation_VarPop
- type AggregationFunction_Operation_VarSamp
- type ApproxCountDistinctFunction
- type ApproxPercentileFunction
- type AuthConfig
- type AuthConfig_AuthConfig_MtlsConfig
- type AuthConfig_AuthConfig_UcServiceCredentialName
- type AvgFunction
- type BackfillSource
- type BackfillSource_BackfillSource_DeltaTableName
- type BackfillSource_BackfillSource_DeltaTableSource
- type BatchCreateMaterializedFeaturesRequest
- type BatchCreateMaterializedFeaturesResponse
- type Client
- func (c *Client) BatchCreateMaterializedFeatures(ctx context.Context, req *BatchCreateMaterializedFeaturesRequest, ...) (*BatchCreateMaterializedFeaturesResponse, error)
- func (c *Client) CreateFeature(ctx context.Context, req *CreateFeatureRequest, opts ...call.Option) (*Feature, error)
- func (c *Client) CreateKafkaConfig(ctx context.Context, req *CreateKafkaConfigRequest, opts ...call.Option) (*KafkaConfig, error)
- func (c *Client) CreateMaterializedFeature(ctx context.Context, req *CreateMaterializedFeatureRequest, ...) (*MaterializedFeature, error)
- func (c *Client) CreateStream(ctx context.Context, req *CreateStreamRequest, opts ...call.Option) (*Stream, error)
- func (c *Client) DeleteFeature(ctx context.Context, req *DeleteFeatureRequest, opts ...call.Option) error
- func (c *Client) DeleteKafkaConfig(ctx context.Context, req *DeleteKafkaConfigRequest, opts ...call.Option) error
- func (c *Client) DeleteMaterializedFeature(ctx context.Context, req *DeleteMaterializedFeatureRequest, ...) error
- func (c *Client) DeleteStream(ctx context.Context, req *DeleteStreamRequest, opts ...call.Option) error
- func (c *Client) GetFeature(ctx context.Context, req *GetFeatureRequest, opts ...call.Option) (*Feature, error)
- func (c *Client) GetKafkaConfig(ctx context.Context, req *GetKafkaConfigRequest, opts ...call.Option) (*KafkaConfig, error)
- func (c *Client) GetMaterializedFeature(ctx context.Context, req *GetMaterializedFeatureRequest, opts ...call.Option) (*MaterializedFeature, error)
- func (c *Client) GetStream(ctx context.Context, req *GetStreamRequest, opts ...call.Option) (*Stream, error)
- func (c *Client) ListFeatures(ctx context.Context, req *ListFeaturesRequest, opts ...call.Option) (*ListFeaturesResponse, error)
- func (c *Client) ListFeaturesIter(ctx context.Context, req *ListFeaturesRequest, opts ...call.Option) iter.Seq2[*Feature, error]
- func (c *Client) ListKafkaConfigs(ctx context.Context, req *ListKafkaConfigsRequest, opts ...call.Option) (*ListKafkaConfigsResponse, error)
- func (c *Client) ListKafkaConfigsIter(ctx context.Context, req *ListKafkaConfigsRequest, opts ...call.Option) iter.Seq2[*KafkaConfig, error]
- func (c *Client) ListMaterializedFeatures(ctx context.Context, req *ListMaterializedFeaturesRequest, opts ...call.Option) (*ListMaterializedFeaturesResponse, error)
- func (c *Client) ListMaterializedFeaturesIter(ctx context.Context, req *ListMaterializedFeaturesRequest, opts ...call.Option) iter.Seq2[*MaterializedFeature, error]
- func (c *Client) ListStreams(ctx context.Context, req *ListStreamsRequest, opts ...call.Option) (*ListStreamsResponse, error)
- func (c *Client) ListStreamsIter(ctx context.Context, req *ListStreamsRequest, opts ...call.Option) iter.Seq2[*Stream, error]
- func (c *Client) UpdateFeature(ctx context.Context, req *UpdateFeatureRequest, opts ...call.Option) (*Feature, error)
- func (c *Client) UpdateKafkaConfig(ctx context.Context, req *UpdateKafkaConfigRequest, opts ...call.Option) (*KafkaConfig, error)
- func (c *Client) UpdateMaterializedFeature(ctx context.Context, req *UpdateMaterializedFeatureRequest, ...) (*MaterializedFeature, error)
- func (c *Client) UpdateStream(ctx context.Context, req *UpdateStreamRequest, opts ...call.Option) (*Stream, error)
- type ColumnSelection
- type CountFunction
- type CreateFeatureRequest
- type CreateKafkaConfigRequest
- type CreateMaterializedFeatureRequest
- type CreateStreamRequest
- type CronSchedule
- type CustomUdf
- type DataSource
- type DataSource_DataSource_DeltaTableSource
- type DataSource_DataSource_KafkaSource
- type DataSource_DataSource_RequestSource
- type DataSource_DataSource_StreamSource
- type DeleteFeatureRequest
- type DeleteKafkaConfigRequest
- type DeleteMaterializedFeatureRequest
- type DeleteStreamRequest
- type DeltaTableSource
- type DirectMtlsConfig
- type DirectSchemas
- type EntityColumn
- type Feature
- type FieldDefinition
- type FirstDistinctFunction
- type FirstFunction
- type FirstNFunction
- type FlatSchema
- type Function
- type Function_Function_AggregationFunction
- type Function_Function_ColumnSelection
- type Function_Function_CustomUdf
- type GetFeatureRequest
- type GetKafkaConfigRequest
- type GetMaterializedFeatureRequest
- type GetStreamRequest
- type IngestionConfig
- type IngestionDestination
- type IngestionDestination_IngestionDestination_DeltaTableName
- type InputBinding
- type JobContext
- type KafkaConfig
- type KafkaSource
- type KafkaStreamConfig
- type KafkaSubscriptionMode
- type KafkaSubscriptionMode_SubscriptionMode_Assign
- type KafkaSubscriptionMode_SubscriptionMode_Subscribe
- type KafkaSubscriptionMode_SubscriptionMode_SubscribePattern
- type KinesisStreamConfig
- type KinesisStreamConfig_StreamIdentifier_StreamArns
- type KinesisStreamConfig_StreamIdentifier_StreamNames
- type LastDistinctFunction
- type LastFunction
- type LastNFunction
- type LineageContext
- type ListFeaturesRequest
- type ListFeaturesResponse
- type ListKafkaConfigsRequest
- type ListKafkaConfigsResponse
- type ListMaterializedFeaturesRequest
- type ListMaterializedFeaturesResponse
- type ListStreamsRequest
- type ListStreamsResponse
- type MaterializedFeature
- type MaterializedFeature_Destination_OfflineStoreConfig
- type MaterializedFeature_Destination_OnlineStoreConfig
- type MaterializedFeature_PipelineScheduleState
- type MaterializedFeature_Trigger_CronScheduleTrigger
- type MaterializedFeature_Trigger_StreamingMode
- type MaterializedFeature_Trigger_TableTrigger
- type MaxFunction
- type MinFunction
- type MtlsConfig
- type OfflineStoreConfig
- type OnlineStoreConfig
- type ProtoSchemaSpec
- type RequestSource
- type RequestSource_Schema_FlatSchema
- type RollingWindow
- type SawtoothWindow
- type ScalarDataType
- type SchemaConfig
- type SchemaConfig_Schema_AvroSchema
- type SchemaConfig_Schema_JsonSchema
- type SchemaConfig_Schema_ProtoSchema
- type SchemaLocator
- type SchemaLocator_ConfluentSchema
- type SchemaLocator_Format
- type SchemaLocator_RegistrySchema_ConfluentSchema
- type SchemaRegistryConfig
- type SecretScopeReference
- type SlidingWindow
- type SourceLateness
- type StddevPopFunction
- type StddevSampFunction
- type Stream
- type StreamArnList
- type StreamConnectionConfig
- type StreamConnectionConfig_ConnectionConfig_DirectMtlsConfig
- type StreamConnectionConfig_ConnectionConfig_UcConnectionName
- type StreamNameList
- type StreamSchemaConfig
- type StreamSchemaConfig_SchemaConfig_DirectSchemas
- type StreamSchemaConfig_SchemaConfig_SchemaRegistryConfig
- type StreamSource
- type StreamSourceConfig
- type StreamSourceConfig_SourceConfig_KafkaStreamConfig
- type StreamSourceConfig_SourceConfig_KinesisStreamConfig
- type StreamingMode
- type StreamingMode_StreamingModeType
- type SubscriptionModedeprecated
- type SubscriptionMode_SubscriptionMode_Assign
- type SubscriptionMode_SubscriptionMode_Subscribe
- type SubscriptionMode_SubscriptionMode_SubscribePattern
- type SumFunction
- type TableTrigger
- type TimeWindow
- type TimeWindow_WindowType_Rolling
- type TimeWindow_WindowType_Sawtooth
- type TimeWindow_WindowType_Sliding
- type TimeWindow_WindowType_Tumbling
- type TimeseriesColumn
- type TumblingWindow
- type UpdateFeatureRequest
- type UpdateKafkaConfigRequest
- type UpdateMaterializedFeatureRequest
- type UpdateStreamRequest
- type VarPopFunction
- type VarSampFunction
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type AggregationFunction ¶
type AggregationFunction struct {
// The type of the aggregation function.
Operation isAggregationFunction_Operation
// The time window over which the aggregation is computed.
TimeWindow *TimeWindow `fieldmask:"time_window"`
// contains filtered or unexported fields
}
An aggregation function applied over a time window..
type AggregationFunction_Operation_ApproxCountDistinct ¶
type AggregationFunction_Operation_ApproxCountDistinct struct {
ApproxCountDistinct ApproxCountDistinctFunction `fieldmask:"approx_count_distinct"`
}
AggregationFunction_Operation_ApproxCountDistinct selects ApproxCountDistinct for AggregationFunction.Operation.
type AggregationFunction_Operation_ApproxPercentile ¶
type AggregationFunction_Operation_ApproxPercentile struct {
ApproxPercentile ApproxPercentileFunction `fieldmask:"approx_percentile"`
}
AggregationFunction_Operation_ApproxPercentile selects ApproxPercentile for AggregationFunction.Operation.
type AggregationFunction_Operation_Avg ¶
type AggregationFunction_Operation_Avg struct {
Avg AvgFunction `fieldmask:"avg"`
}
AggregationFunction_Operation_Avg selects Avg for AggregationFunction.Operation.
type AggregationFunction_Operation_CountFunction ¶
type AggregationFunction_Operation_CountFunction struct {
CountFunction CountFunction `fieldmask:"count_function"`
}
AggregationFunction_Operation_CountFunction selects CountFunction for AggregationFunction.Operation.
type AggregationFunction_Operation_First ¶
type AggregationFunction_Operation_First struct {
First FirstFunction `fieldmask:"first"`
}
AggregationFunction_Operation_First selects First for AggregationFunction.Operation.
type AggregationFunction_Operation_FirstDistinct ¶
type AggregationFunction_Operation_FirstDistinct struct {
FirstDistinct FirstDistinctFunction `fieldmask:"first_distinct"`
}
AggregationFunction_Operation_FirstDistinct selects FirstDistinct for AggregationFunction.Operation.
type AggregationFunction_Operation_FirstN ¶
type AggregationFunction_Operation_FirstN struct {
FirstN FirstNFunction `fieldmask:"first_n"`
}
AggregationFunction_Operation_FirstN selects FirstN for AggregationFunction.Operation.
type AggregationFunction_Operation_Last ¶
type AggregationFunction_Operation_Last struct {
Last LastFunction `fieldmask:"last"`
}
AggregationFunction_Operation_Last selects Last for AggregationFunction.Operation.
type AggregationFunction_Operation_LastDistinct ¶
type AggregationFunction_Operation_LastDistinct struct {
LastDistinct LastDistinctFunction `fieldmask:"last_distinct"`
}
AggregationFunction_Operation_LastDistinct selects LastDistinct for AggregationFunction.Operation.
type AggregationFunction_Operation_LastN ¶
type AggregationFunction_Operation_LastN struct {
LastN LastNFunction `fieldmask:"last_n"`
}
AggregationFunction_Operation_LastN selects LastN for AggregationFunction.Operation.
type AggregationFunction_Operation_Max ¶
type AggregationFunction_Operation_Max struct {
Max MaxFunction `fieldmask:"max"`
}
AggregationFunction_Operation_Max selects Max for AggregationFunction.Operation.
type AggregationFunction_Operation_Min ¶
type AggregationFunction_Operation_Min struct {
Min MinFunction `fieldmask:"min"`
}
AggregationFunction_Operation_Min selects Min for AggregationFunction.Operation.
type AggregationFunction_Operation_StddevPop ¶
type AggregationFunction_Operation_StddevPop struct {
StddevPop StddevPopFunction `fieldmask:"stddev_pop"`
}
AggregationFunction_Operation_StddevPop selects StddevPop for AggregationFunction.Operation.
type AggregationFunction_Operation_StddevSamp ¶
type AggregationFunction_Operation_StddevSamp struct {
StddevSamp StddevSampFunction `fieldmask:"stddev_samp"`
}
AggregationFunction_Operation_StddevSamp selects StddevSamp for AggregationFunction.Operation.
type AggregationFunction_Operation_Sum ¶
type AggregationFunction_Operation_Sum struct {
Sum SumFunction `fieldmask:"sum"`
}
AggregationFunction_Operation_Sum selects Sum for AggregationFunction.Operation.
type AggregationFunction_Operation_VarPop ¶
type AggregationFunction_Operation_VarPop struct {
VarPop VarPopFunction `fieldmask:"var_pop"`
}
AggregationFunction_Operation_VarPop selects VarPop for AggregationFunction.Operation.
type AggregationFunction_Operation_VarSamp ¶
type AggregationFunction_Operation_VarSamp struct {
VarSamp VarSampFunction `fieldmask:"var_samp"`
}
AggregationFunction_Operation_VarSamp selects VarSamp for AggregationFunction.Operation.
type ApproxCountDistinctFunction ¶
type ApproxCountDistinctFunction struct {
// The input column from which the approximate count of distinct values is
// computed.
Input *string `fieldmask:"input"`
// The maximum relative standard deviation allowed (default defined by Spark).
RelativeSd *float64 `fieldmask:"relative_sd"`
}
Computes the approximate count of distinct values..
type ApproxPercentileFunction ¶
type ApproxPercentileFunction struct {
// The input column from which the approximate percentile is computed.
Input *string `fieldmask:"input"`
// The percentile value to compute (between 0 and 1).
Percentile *float64 `fieldmask:"percentile"`
// The accuracy parameter (higher is more accurate but slower).
Accuracy *int64 `fieldmask:"accuracy"`
}
Computes the approximate percentile of values..
type AuthConfig ¶
type AuthConfig struct {
AuthConfig isAuthConfig_AuthConfig
// contains filtered or unexported fields
}
type AuthConfig_AuthConfig_MtlsConfig ¶
type AuthConfig_AuthConfig_MtlsConfig struct {
MtlsConfig MtlsConfig `fieldmask:"mtls_config"`
}
AuthConfig_AuthConfig_MtlsConfig selects MtlsConfig for AuthConfig.AuthConfig. Mutual-TLS authentication. See MtlsConfig.
type AuthConfig_AuthConfig_UcServiceCredentialName ¶
type AuthConfig_AuthConfig_UcServiceCredentialName struct {
UcServiceCredentialName string `fieldmask:"uc_service_credential_name"`
}
AuthConfig_AuthConfig_UcServiceCredentialName selects UcServiceCredentialName for AuthConfig.AuthConfig. Name of the Unity Catalog service credential. This value will be set under the option databricks.serviceCredential
type AvgFunction ¶
type AvgFunction struct {
// The input column from which the average is computed. For Kafka sources, use
// dot-prefixed path notation (e.g., "value.amount"). For nested fields, the
// leaf node name is used. Colon-prefixed notation (e.g., "value:amount") is
// supported for backwards compatibility but is deprecated; migrate to dot
// notation.
Input *string `fieldmask:"input"`
}
Computes the average of values..
type BackfillSource ¶
type BackfillSource struct {
BackfillSource isBackfillSource_BackfillSource
// contains filtered or unexported fields
}
type BackfillSource_BackfillSource_DeltaTableName ¶
type BackfillSource_BackfillSource_DeltaTableName struct {
DeltaTableName string `fieldmask:"delta_table_name"`
}
BackfillSource_BackfillSource_DeltaTableName selects DeltaTableName for BackfillSource.BackfillSource. The full three-part name (catalog, schema, name) of the Delta table containing the historical data to backfill.
type BackfillSource_BackfillSource_DeltaTableSource ¶
type BackfillSource_BackfillSource_DeltaTableSource struct {
DeltaTableSource DeltaTableSource `fieldmask:"delta_table_source"`
}
BackfillSource_BackfillSource_DeltaTableSource selects DeltaTableSource for BackfillSource.BackfillSource. Deprecated: Use delta_table_name instead. Kept for backwards compatibility. The Delta table source containing the historical data to backfill. Only the delta table name is used for backfill, other fields are ignored.
type BatchCreateMaterializedFeaturesRequest ¶
type BatchCreateMaterializedFeaturesRequest struct {
// The requests to create materialized features.
Requests []CreateMaterializedFeatureRequest
}
type BatchCreateMaterializedFeaturesResponse ¶
type BatchCreateMaterializedFeaturesResponse struct {
// The created materialized features with assigned IDs.
MaterializedFeatures []MaterializedFeature
}
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
func (*Client) BatchCreateMaterializedFeatures ¶
func (c *Client) BatchCreateMaterializedFeatures(ctx context.Context, req *BatchCreateMaterializedFeaturesRequest, opts ...call.Option) (*BatchCreateMaterializedFeaturesResponse, error)
Batch create materialized features.
func (*Client) CreateFeature ¶
func (c *Client) CreateFeature(ctx context.Context, req *CreateFeatureRequest, opts ...call.Option) (*Feature, error)
Create a Feature.
func (*Client) CreateKafkaConfig ¶
func (c *Client) CreateKafkaConfig(ctx context.Context, req *CreateKafkaConfigRequest, opts ...call.Option) (*KafkaConfig, error)
Create a Kafka config. During PrPr, Kafka configs can be read and used when creating features under the entire metastore. Only the creator of the Kafka config can delete it.
func (*Client) CreateMaterializedFeature ¶
func (c *Client) CreateMaterializedFeature(ctx context.Context, req *CreateMaterializedFeatureRequest, opts ...call.Option) (*MaterializedFeature, error)
Create a materialized feature.
func (*Client) CreateStream ¶
func (c *Client) CreateStream(ctx context.Context, req *CreateStreamRequest, opts ...call.Option) (*Stream, error)
Create a Stream, a governed UC entity representing an external streaming data source.
func (*Client) DeleteFeature ¶
func (c *Client) DeleteFeature(ctx context.Context, req *DeleteFeatureRequest, opts ...call.Option) error
Delete a Feature.
func (*Client) DeleteKafkaConfig ¶
func (c *Client) DeleteKafkaConfig(ctx context.Context, req *DeleteKafkaConfigRequest, opts ...call.Option) error
Delete a Kafka config. During PrPr, Kafka configs can be read and used when creating features under the entire metastore. Only the creator of the Kafka config can delete it.
func (*Client) DeleteMaterializedFeature ¶
func (c *Client) DeleteMaterializedFeature(ctx context.Context, req *DeleteMaterializedFeatureRequest, opts ...call.Option) error
Delete a materialized feature.
func (*Client) DeleteStream ¶
func (c *Client) DeleteStream(ctx context.Context, req *DeleteStreamRequest, opts ...call.Option) error
Delete a Stream by its full three-part name (catalog.schema.stream).
func (*Client) GetFeature ¶
func (c *Client) GetFeature(ctx context.Context, req *GetFeatureRequest, opts ...call.Option) (*Feature, error)
Get a Feature.
func (*Client) GetKafkaConfig ¶
func (c *Client) GetKafkaConfig(ctx context.Context, req *GetKafkaConfigRequest, opts ...call.Option) (*KafkaConfig, error)
Get a Kafka config. During PrPr, Kafka configs can be read and used when creating features under the entire metastore. Only the creator of the Kafka config can delete it.
func (*Client) GetMaterializedFeature ¶
func (c *Client) GetMaterializedFeature(ctx context.Context, req *GetMaterializedFeatureRequest, opts ...call.Option) (*MaterializedFeature, error)
Get a materialized feature.
func (*Client) GetStream ¶
func (c *Client) GetStream(ctx context.Context, req *GetStreamRequest, opts ...call.Option) (*Stream, error)
Get a Stream by its full three-part name (catalog.schema.stream).
func (*Client) ListFeatures ¶
func (c *Client) ListFeatures(ctx context.Context, req *ListFeaturesRequest, opts ...call.Option) (*ListFeaturesResponse, error)
List Features.
func (*Client) ListFeaturesIter ¶
func (c *Client) ListFeaturesIter(ctx context.Context, req *ListFeaturesRequest, opts ...call.Option) iter.Seq2[*Feature, error]
ListFeaturesIter returns an iterator that iterates over the results of ListFeatures.
For example:
for item, err := range c.ListFeaturesIter(ctx, &ListFeaturesRequest{}) {
if err != nil {
return err
}
fmt.Println(item)
}
Options opts are passed to each ListFeatures call made by the iterator under the hood.
Callers who need custom pagination logic should use ListFeatures directly.
func (*Client) ListKafkaConfigs ¶
func (c *Client) ListKafkaConfigs(ctx context.Context, req *ListKafkaConfigsRequest, opts ...call.Option) (*ListKafkaConfigsResponse, error)
List Kafka configs. During PrPr, Kafka configs can be read and used when creating features under the entire metastore. Only the creator of the Kafka config can delete it.
func (*Client) ListKafkaConfigsIter ¶
func (c *Client) ListKafkaConfigsIter(ctx context.Context, req *ListKafkaConfigsRequest, opts ...call.Option) iter.Seq2[*KafkaConfig, error]
ListKafkaConfigsIter returns an iterator that iterates over the results of ListKafkaConfigs.
For example:
for item, err := range c.ListKafkaConfigsIter(ctx, &ListKafkaConfigsRequest{}) {
if err != nil {
return err
}
fmt.Println(item)
}
Options opts are passed to each ListKafkaConfigs call made by the iterator under the hood.
Callers who need custom pagination logic should use ListKafkaConfigs directly.
func (*Client) ListMaterializedFeatures ¶
func (c *Client) ListMaterializedFeatures(ctx context.Context, req *ListMaterializedFeaturesRequest, opts ...call.Option) (*ListMaterializedFeaturesResponse, error)
List materialized features.
func (*Client) ListMaterializedFeaturesIter ¶
func (c *Client) ListMaterializedFeaturesIter(ctx context.Context, req *ListMaterializedFeaturesRequest, opts ...call.Option) iter.Seq2[*MaterializedFeature, error]
ListMaterializedFeaturesIter returns an iterator that iterates over the results of ListMaterializedFeatures.
For example:
for item, err := range c.ListMaterializedFeaturesIter(ctx, &ListMaterializedFeaturesRequest{}) {
if err != nil {
return err
}
fmt.Println(item)
}
Options opts are passed to each ListMaterializedFeatures call made by the iterator under the hood.
Callers who need custom pagination logic should use ListMaterializedFeatures directly.
func (*Client) ListStreams ¶
func (c *Client) ListStreams(ctx context.Context, req *ListStreamsRequest, opts ...call.Option) (*ListStreamsResponse, error)
List Streams under a given catalog.schema parent.
func (*Client) ListStreamsIter ¶
func (c *Client) ListStreamsIter(ctx context.Context, req *ListStreamsRequest, opts ...call.Option) iter.Seq2[*Stream, error]
ListStreamsIter returns an iterator that iterates over the results of ListStreams.
For example:
for item, err := range c.ListStreamsIter(ctx, &ListStreamsRequest{}) {
if err != nil {
return err
}
fmt.Println(item)
}
Options opts are passed to each ListStreams call made by the iterator under the hood.
Callers who need custom pagination logic should use ListStreams directly.
func (*Client) UpdateFeature ¶
func (c *Client) UpdateFeature(ctx context.Context, req *UpdateFeatureRequest, opts ...call.Option) (*Feature, error)
Update a Feature.
func (*Client) UpdateKafkaConfig ¶
func (c *Client) UpdateKafkaConfig(ctx context.Context, req *UpdateKafkaConfigRequest, opts ...call.Option) (*KafkaConfig, error)
Update a Kafka config. During PrPr, Kafka configs can be read and used when creating features under the entire metastore. Only the creator of the Kafka config can delete it.
func (*Client) UpdateMaterializedFeature ¶
func (c *Client) UpdateMaterializedFeature(ctx context.Context, req *UpdateMaterializedFeatureRequest, opts ...call.Option) (*MaterializedFeature, error)
Update a materialized feature (pause/resume).
func (*Client) UpdateStream ¶
func (c *Client) UpdateStream(ctx context.Context, req *UpdateStreamRequest, opts ...call.Option) (*Stream, error)
Update a Stream. Only fields listed in `update_mask` are mutated.
type ColumnSelection ¶
type ColumnSelection struct {
// Column name from source to select as the feature value.
Column *string `fieldmask:"column"`
}
A ColumnSelection function, equivalent to the LAST() record of an entity over a lifetime window.
type CountFunction ¶
type CountFunction struct {
// The input column from which the count is computed. For Kafka sources, use
// dot-prefixed path notation (e.g., "value.amount"). For nested fields, the
// leaf node name is used. Colon-prefixed notation (e.g., "value:amount") is
// supported for backwards compatibility but is deprecated; migrate to dot
// notation.
Input *string `fieldmask:"input"`
}
Computes the count of values..
type CreateFeatureRequest ¶
type CreateFeatureRequest struct {
// Feature to create.
Feature *Feature
}
type CreateKafkaConfigRequest ¶
type CreateKafkaConfigRequest struct {
KafkaConfig *KafkaConfig
}
type CreateMaterializedFeatureRequest ¶
type CreateMaterializedFeatureRequest struct {
// The materialized feature to create.
MaterializedFeature *MaterializedFeature
}
type CreateStreamRequest ¶
type CreateStreamRequest struct {
// The Stream to create.
Stream *Stream
}
Create a Stream, a governed UC entity representing an external streaming data source..
type CronSchedule ¶
type CronSchedule struct {
// The cron expression defining the schedule (e.g., "0 0 * * *" for daily at
// midnight).
CronExpression *string `fieldmask:"cron_expression"`
}
A cron-based schedule trigger for the materialization pipeline..
type CustomUdf ¶
type CustomUdf struct {
// Fully qualified 3-part Unity Catalog path of the function to apply.
FunctionPath *string `fieldmask:"function_path"`
// Binds each UC function parameter to a source column. May be empty for
// zero-argument functions (e.g. a timestamp generator).
InputBindings []InputBinding `fieldmask:"input_bindings"`
}
A CustomUdf function applies a registered Unity Catalog function row-wise to source columns, producing a single output column per row..
type DataSource ¶
type DataSource struct {
DataSource isDataSource_DataSource
// Completeness timing for this Feature's use of the source. This configuration
// is part of the Feature definition; it does not modify the underlying table or
// stream.
Lateness *SourceLateness `fieldmask:"lateness"`
// contains filtered or unexported fields
}
Specifies the data source backing a feature. Exactly one source type must be set..
type DataSource_DataSource_DeltaTableSource ¶
type DataSource_DataSource_DeltaTableSource struct {
DeltaTableSource DeltaTableSource `fieldmask:"delta_table_source"`
}
DataSource_DataSource_DeltaTableSource selects DeltaTableSource for DataSource.DataSource. A Delta table data source.
type DataSource_DataSource_KafkaSource ¶
type DataSource_DataSource_KafkaSource struct {
KafkaSource KafkaSource `fieldmask:"kafka_source"`
}
DataSource_DataSource_KafkaSource selects KafkaSource for DataSource.DataSource. A Kafka stream data source.
type DataSource_DataSource_RequestSource ¶
type DataSource_DataSource_RequestSource struct {
RequestSource RequestSource `fieldmask:"request_source"`
}
DataSource_DataSource_RequestSource selects RequestSource for DataSource.DataSource. A request-time data source.
type DataSource_DataSource_StreamSource ¶
type DataSource_DataSource_StreamSource struct {
StreamSource StreamSource `fieldmask:"stream_source"`
}
DataSource_DataSource_StreamSource selects StreamSource for DataSource.DataSource. A Stream data source.
type DeleteFeatureRequest ¶
type DeleteFeatureRequest struct {
// Name of the feature to delete.
FullName *string
}
type DeleteKafkaConfigRequest ¶
type DeleteKafkaConfigRequest struct {
// Name of the Kafka config to delete.
Name *string
}
type DeleteMaterializedFeatureRequest ¶
type DeleteMaterializedFeatureRequest struct {
// The ID of the materialized feature to delete.
MaterializedFeatureId *string
}
type DeleteStreamRequest ¶
type DeleteStreamRequest struct {
// Full three-part name (catalog.schema.stream) of the Stream to delete.
Name *string
}
Delete a Stream by its full three-part name (catalog.schema.stream)..
type DeltaTableSource ¶
type DeltaTableSource struct {
// The full three-part (catalog, schema, table) name of the Delta table.
FullName *string `fieldmask:"full_name"`
// Single WHERE clause to filter delta table before applying transformations.
// Will be row-wise evaluated, so should only include conditionals and
// projections.
FilterCondition *string `fieldmask:"filter_condition"`
// A single SQL SELECT expression applied after filter_condition. Should
// contains all the columns needed (eg. "SELECT *, col_a + col_b AS col_c FROM
// x.y.z WHERE col_a > 0" would have `transformation_sql` "*, col_a + col_b AS
// col_c") If transformation_sql is not provided, all columns of the delta table
// are present in the DataSource dataframe.
TransformationSql *string `fieldmask:"transformation_sql"`
// Schema of the resulting dataframe after transformations, in Spark StructType
// JSON format (from df.schema.json()). Required if transformation_sql is
// specified. Example:
// {"type":"struct","fields":[{"name":"col_a","type":"integer","nullable":true,"metadata":{}},{"name":"col_c","type":"integer","nullable":true,"metadata":{}}]}
DataframeSchema *string `fieldmask:"dataframe_schema"`
}
type DirectMtlsConfig ¶
type DirectMtlsConfig struct {
// A comma-separated list of host:port pairs for the Kafka bootstrap servers.
BootstrapServers *string `fieldmask:"bootstrap_servers"`
// Mutual-TLS authentication configuration.
MtlsConfig *MtlsConfig `fieldmask:"mtls_config"`
}
Direct connection configs for mTLS, as Kafka Connections do not support mTLS yet . Temporarily used until UC Kafka Connections gain mTLS support..
type DirectSchemas ¶
type DirectSchemas struct {
// Schema for the message payload. For Kafka, this is the value schema. Unless
// the platform supports another schema (e.g. keys for Kafka), this must be
// specified.
PayloadSchema *SchemaConfig `fieldmask:"payload_schema"`
// Schema for the message key. This is only used for Kafka streams. For Kafka,
// at least one of payload_schema or key_schema must be specified.
KeySchema *SchemaConfig `fieldmask:"key_schema"`
}
Schema definitions provided directly on the Stream, as opposed to referencing a schema registry. To resolve schemas from a registry instead, use SchemaRegistryConfig..
type EntityColumn ¶
type EntityColumn struct {
// The name of the entity column. For Kafka sources, use dot-prefixed path
// notation to reference fields within the key or value schema (e.g.,
// "value.user_id", "key.partition_key"). For nested fields, the leaf node name
// (e.g., "user_id" from "value.trip_details.user_id") is what will be present
// in materialized tables and expected to match at query time. Colon-prefixed
// notation (e.g., "value:user_id") is supported for backwards compatibility but
// is deprecated; migrate to dot notation.
Name *string
}
type Feature ¶
type Feature struct {
// The full three-part name (catalog, schema, name) of the feature. This is the
// feature's resource identifier; the catalog_name, schema_name, and name fields
// below are OUTPUT_ONLY decomposed views of this value.
FullName *string `fieldmask:"full_name"`
// The data source of the feature.
Source *DataSource `fieldmask:"source"`
// The function by which the feature is computed.
Function *Function `fieldmask:"function"`
// The description of the feature.
Description *string `fieldmask:"description"`
// Lineage context information for this feature. WARNING: This field is
// primarily intended for internal use by <Databricks> systems and is
// automatically populated when features are created through <Databricks>
// notebooks or jobs. Users should not manually set this field as incorrect
// values may lead to inaccurate lineage tracking or unexpected behavior. This
// field will be set by feature-engineering client and should be left unset by
// SDK and terraform users.
LineageContext *LineageContext `fieldmask:"lineage_context"`
// The entity columns for the feature, used as aggregation keys and for
// query-time lookup.
Entities []EntityColumn `fieldmask:"entities"`
// Column recording time, used for point-in-time joins, backfills, and
// aggregations.
TimeseriesColumn *TimeseriesColumn `fieldmask:"timeseries_column"`
// Name of parent catalog.
CatalogName *string `fieldmask:"catalog_name"`
// Name of parent schema relative to its parent catalog.
SchemaName *string `fieldmask:"schema_name"`
// Name of the feature, extracted from the full three-part name
// (catalog.schema.name).
Name *string `fieldmask:"name"`
// Time at which this feature was created.
CreatedAt *types.Time `fieldmask:"created_at"`
// Username of the feature creator.
CreatedBy *string `fieldmask:"created_by"`
}
type FieldDefinition ¶
type FieldDefinition struct {
// The name of the field.
Name *string
// The scalar data type of the field.
DataType ScalarDataType
}
A single field definition within a FlatSchema, specifying the field name and its scalar data type. Does not support nested or complex types (arrays, maps, structs)..
type FirstDistinctFunction ¶
type FirstDistinctFunction struct {
// The input column from which the first N distinct values are returned.
Input *string `fieldmask:"input"`
// The number of distinct values to return.
N *int64 `fieldmask:"n"`
}
Returns the first N distinct values, ordered by the feature's timeseries column..
type FirstFunction ¶
type FirstFunction struct {
// The input column from which the first value is returned.
Input *string `fieldmask:"input"`
}
Returns the first value..
type FirstNFunction ¶
type FirstNFunction struct {
// The input column from which the first N values are returned.
Input *string `fieldmask:"input"`
// The number of values to return.
N *int64 `fieldmask:"n"`
}
Returns the first N values, ordered by the feature's timeseries column..
type FlatSchema ¶
type FlatSchema struct {
// The list of fields in this schema.
Fields []FieldDefinition `fieldmask:"fields"`
}
A flat (non-nested) schema for request-time fields, defined as an ordered list of field definitions. This schema only supports scalar types..
type Function ¶
type Function struct {
Function isFunction_Function
// contains filtered or unexported fields
}
type Function_Function_AggregationFunction ¶
type Function_Function_AggregationFunction struct {
AggregationFunction AggregationFunction `fieldmask:"aggregation_function"`
}
Function_Function_AggregationFunction selects AggregationFunction for Function.Function. An aggregation function applied over a time window.
type Function_Function_ColumnSelection ¶
type Function_Function_ColumnSelection struct {
ColumnSelection ColumnSelection `fieldmask:"column_selection"`
}
Function_Function_ColumnSelection selects ColumnSelection for Function.Function. Selects the latest value of a single column in a data source
type Function_Function_CustomUdf ¶
type Function_Function_CustomUdf struct {
CustomUdf CustomUdf `fieldmask:"custom_udf"`
}
Function_Function_CustomUdf selects CustomUdf for Function.Function. Applies a registered Unity Catalog function row-wise to source columns.
type GetFeatureRequest ¶
type GetFeatureRequest struct {
// Name of the feature to get.
FullName *string
}
type GetKafkaConfigRequest ¶
type GetKafkaConfigRequest struct {
// Name of the Kafka config to get.
Name *string
}
type GetMaterializedFeatureRequest ¶
type GetMaterializedFeatureRequest struct {
// The ID of the materialized feature.
MaterializedFeatureId *string
}
type GetStreamRequest ¶
type GetStreamRequest struct {
// Full three-part name (catalog.schema.stream) of the Stream to get.
Name *string
}
Get a Stream by its full three-part name (catalog.schema.stream)..
type IngestionConfig ¶
type IngestionConfig struct {
// Destination for the <Databricks>-managed Delta table that holds an offline
// copy of the streaming data for querying and training. This table contains
// both 1) forward-filled data from the Stream and 2) backfilled data from the
// BackfillSource (if provided). This table is created and managed by
// <Databricks> and is deleted when the Stream is deleted.
IngestionDestination *IngestionDestination `fieldmask:"ingestion_destination"`
// A user-provided source for backfilling data. Historical data is used when
// creating a training set from streaming features linked to this Stream. The
// backfill data stored in this location will be copied into the ingestion table
// for offline querying and training. The schema for this source must match
// exactly that of the key and payload schemas specified for this Stream.
BackfillSource *BackfillSource `fieldmask:"backfill_source"`
// Column paths used to identify duplicate rows during ingestion; only one row
// per distinct combination of these values is kept. Use dot notation for nested
// fields (e.g. `value.user_id`). Empty list means every column is compared.
DeduplicationColumns []string `fieldmask:"deduplication_columns"`
// The ID of the SDP pipeline that continuously copies new events from the
// streaming source into the ingestion Delta table.
IngestionPipelineId *string `fieldmask:"ingestion_pipeline_id"`
// The ID of the Databricks Job that performs the forward-fill ingestion.
IngestionJobId *int64 `fieldmask:"ingestion_job_id"`
// The ID of the Databricks Job that performs the historical backfill of the
// ingestion Delta table.
BackfillJobId *int64 `fieldmask:"backfill_job_id"`
}
Configuration for the <Databricks>-managed ingestion pipeline. Groups the ingestion destination (required) and optional backfill source..
type IngestionDestination ¶
type IngestionDestination struct {
IngestionDestination isIngestionDestination_IngestionDestination
// contains filtered or unexported fields
}
Destination for the <Databricks>-managed Delta table that holds an offline copy of the streaming data for querying and training..
type IngestionDestination_IngestionDestination_DeltaTableName ¶
type IngestionDestination_IngestionDestination_DeltaTableName struct {
DeltaTableName string `fieldmask:"delta_table_name"`
}
IngestionDestination_IngestionDestination_DeltaTableName selects DeltaTableName for IngestionDestination.IngestionDestination. The full three-part name (catalog, schema, name) of the Delta table to be created for ingestion.
type InputBinding ¶
type InputBinding struct {
// Name of the UC function parameter.
Parameter *string
// Source column whose value is passed for this parameter at execution time.
Column *string
}
Binds a single UC function parameter to a source column..
type JobContext ¶
type KafkaConfig ¶
type KafkaConfig struct {
// Name that uniquely identifies this Kafka config within the metastore. This
// will be the identifier used from the Feature object to reference these
// configs for a feature. Can be distinct from topic name.
Name *string `fieldmask:"name"`
// A comma-separated list of host/port pairs pointing to Kafka cluster.
BootstrapServers *string `fieldmask:"bootstrap_servers"`
// Options to configure which Kafka topics to pull data from.
SubscriptionMode *SubscriptionMode `fieldmask:"subscription_mode"`
// Authentication configuration for connection to topics.
AuthConfig *AuthConfig `fieldmask:"auth_config"`
// Schema configuration for extracting message keys from topics. At least one of
// key_schema and value_schema must be provided.
KeySchema *SchemaConfig `fieldmask:"key_schema"`
// Schema configuration for extracting message values from topics. At least one
// of key_schema and value_schema must be provided.
ValueSchema *SchemaConfig `fieldmask:"value_schema"`
// Catch-all for miscellaneous options. Keys should be source options or Kafka
// consumer options (kafka.*)
ExtraOptions map[string]string `fieldmask:"extra_options"`
// A user-provided and managed source for backfilling data. Historical data is
// used when creating a training set from streaming features linked to this
// Kafka config. In the future, a separate table will be maintained by
// <Databricks> for forward filling data. The schema for this source must match
// exactly that of the key and value schemas specified for this Kafka config.
BackfillSource *BackfillSource `fieldmask:"backfill_source"`
// Configuration for ingesting Kafka data into a <Databricks>-managed Delta
// table.
IngestionConfig *IngestionConfig `fieldmask:"ingestion_config"`
}
type KafkaSource ¶
type KafkaSource struct {
// Name of the Kafka source, used to identify it. This is used to look up the
// corresponding KafkaConfig object. Can be distinct from topic name.
Name *string `fieldmask:"name"`
// The filter condition applied to the source data before aggregation.
FilterCondition *string `fieldmask:"filter_condition"`
}
type KafkaStreamConfig ¶
type KafkaStreamConfig struct {
// Options to configure which Kafka topics to pull data from.
SubscriptionMode *KafkaSubscriptionMode `fieldmask:"subscription_mode"`
// Optional Kafka source or consumer options, validated against a server-side
// allowlist at request time. Allowed keys: - `maxOffsetsPerTrigger` -
// `startingOffsets` - `includeHeaders` - `kafka.request.timeout.ms` -
// `kafka.session.timeout.ms` - `kafka.max.partition.fetch.bytes` The following
// keys are ingestion-only and are stripped before being forwarded to the
// materialization pipeline: - `maxOffsetsPerTrigger` - `startingOffsets` Auth
// and connection details belong on the parent Stream's `connection_config`, not
// here.
ExtraOptions map[string]string `fieldmask:"extra_options"`
}
Kafka-specific configuration for a Stream..
type KafkaSubscriptionMode ¶
type KafkaSubscriptionMode struct {
// These match the settings from
// https://spark.apache.org/docs/latest/streaming/structured-streaming-kafka-integration.html
SubscriptionMode isKafkaSubscriptionMode_SubscriptionMode
// contains filtered or unexported fields
}
Subscription mode for Kafka topic selection, matching standard Spark Structured Streaming options..
type KafkaSubscriptionMode_SubscriptionMode_Assign ¶
type KafkaSubscriptionMode_SubscriptionMode_Assign struct {
Assign string `fieldmask:"assign"`
}
KafkaSubscriptionMode_SubscriptionMode_Assign selects Assign for KafkaSubscriptionMode.SubscriptionMode. A JSON string that contains the specific topic-partitions to consume from. For example, for '{"topicA":[0,1],"topicB":[2,4]}', topicA's 0'th and 1st partitions will be consumed from.
type KafkaSubscriptionMode_SubscriptionMode_Subscribe ¶
type KafkaSubscriptionMode_SubscriptionMode_Subscribe struct {
Subscribe string `fieldmask:"subscribe"`
}
KafkaSubscriptionMode_SubscriptionMode_Subscribe selects Subscribe for KafkaSubscriptionMode.SubscriptionMode. A comma-separated list of Kafka topics to read from. For example, 'topicA,topicB,topicC'.
type KafkaSubscriptionMode_SubscriptionMode_SubscribePattern ¶
type KafkaSubscriptionMode_SubscriptionMode_SubscribePattern struct {
SubscribePattern string `fieldmask:"subscribe_pattern"`
}
KafkaSubscriptionMode_SubscriptionMode_SubscribePattern selects SubscribePattern for KafkaSubscriptionMode.SubscriptionMode. A regular expression matching topics to subscribe to. For example, 'topic.*' will subscribe to all topics starting with 'topic'.
type KinesisStreamConfig ¶
type KinesisStreamConfig struct {
// Identifies the Kinesis data stream(s) to read from. Set exactly one of
// stream_names or stream_arns (identify the streams by name or by ARN, but not
// both). A single Stream may read from one or more Kinesis streams.
StreamIdentifier isKinesisStreamConfig_StreamIdentifier
// Optional Kinesis source options, validated against a server-side allowlist at
// request time. Auth and connection details belong on the parent Stream's
// `connection_config`, not here.
ExtraOptions map[string]string `fieldmask:"extra_options"`
// contains filtered or unexported fields
}
Kinesis-specific configuration for a Stream. For the underlying connector and its source options, see the <Databricks> documentation on connecting to Amazon Kinesis (https://docs.databricks.com/aws/en/connect/streaming/kinesis)..
type KinesisStreamConfig_StreamIdentifier_StreamArns ¶
type KinesisStreamConfig_StreamIdentifier_StreamArns struct {
StreamArns StreamArnList `fieldmask:"stream_arns"`
}
KinesisStreamConfig_StreamIdentifier_StreamArns selects StreamArns for KinesisStreamConfig.StreamIdentifier. Kinesis stream ARNs to read from.
type KinesisStreamConfig_StreamIdentifier_StreamNames ¶
type KinesisStreamConfig_StreamIdentifier_StreamNames struct {
StreamNames StreamNameList `fieldmask:"stream_names"`
}
KinesisStreamConfig_StreamIdentifier_StreamNames selects StreamNames for KinesisStreamConfig.StreamIdentifier. Kinesis stream names to read from.
type LastDistinctFunction ¶
type LastDistinctFunction struct {
// The input column from which the last N distinct values are returned.
Input *string `fieldmask:"input"`
// The number of distinct values to return.
N *int64 `fieldmask:"n"`
}
Returns the last N distinct values, ordered by the feature's timeseries column..
type LastFunction ¶
type LastFunction struct {
// The input column from which the last value is returned.
Input *string `fieldmask:"input"`
}
Returns the last value..
type LastNFunction ¶
type LastNFunction struct {
// The input column from which the last N values are returned.
Input *string `fieldmask:"input"`
// The number of values to return.
N *int64 `fieldmask:"n"`
}
Returns the last N values, ordered by the feature's timeseries column..
type LineageContext ¶
type LineageContext struct {
// The notebook ID where this API was invoked.
NotebookId *int64 `fieldmask:"notebook_id"`
// Job context information including job ID and run ID.
JobContext *JobContext `fieldmask:"job_context"`
}
Lineage context information for tracking where an API was invoked. This will allow us to track lineage, which currently uses caller entity information for use across the Lineage Client and Observability in Lumberjack..
type ListFeaturesRequest ¶
type ListFeaturesRequest struct {
// Pagination token to go to the next page based on a previous query.
PageToken *string
// The maximum number of results to return.
PageSize *int
// Name of parent catalog for features of interest.
CatalogName *string
// Name of parent schema relative to its parent catalog.
SchemaName *string
}
Request to list features. Listing is always scoped to a single catalog and schema; catalog_name and schema_name are required..
type ListFeaturesResponse ¶
type ListKafkaConfigsRequest ¶
type ListKafkaConfigsResponse ¶
type ListKafkaConfigsResponse struct {
// List of Kafka configs. Schemas are not included in the response.
KafkaConfigs []KafkaConfig
// Pagination token to request the next page of results for this query.
NextPageToken *string
}
type ListMaterializedFeaturesRequest ¶
type ListMaterializedFeaturesRequest struct {
// Filter by feature name. If specified, only materialized features materialized
// from this feature will be returned.
FeatureName *string
// Pagination token to go to the next page based on a previous query.
PageToken *string
// The maximum number of results to return. Defaults to 100 if not specified.
// Cannot be greater than 1000.
PageSize *int
}
type ListMaterializedFeaturesResponse ¶
type ListMaterializedFeaturesResponse struct {
// List of materialized features.
MaterializedFeatures []MaterializedFeature
// Pagination token to request the next page of results for this query.
NextPageToken *string
}
type ListStreamsRequest ¶
type ListStreamsRequest struct {
// Two-part name (catalog.schema) of the parent under which to list Streams.
Parent *string
// The maximum number of results to return.
PageSize *int
// Pagination token to go to the next page based on a previous query.
PageToken *string
}
List Streams under a given parent.
NOTE: Results are post-filtered by access permission on each stream's ingestion table. This means: - Returned results may be fewer than page_size (including zero) - Page token points to next unfiltered batch, not next filtered batch, and may point to an item that will be filtered out - Callers should paginate until next_page_token is empty to retrieve all accessible streams.
type ListStreamsResponse ¶
type ListStreamsResponse struct {
// List of Streams.
Streams []Stream
// Pagination token to request the next page of results for this query.
NextPageToken *string
}
Response to a ListStreamsRequest.
NOTE: Results are post-filtered by access permission on each stream's ingestion table. This means: - Returned results may be fewer than page_size (including zero) - Page token points to next unfiltered batch, not next filtered batch, and may point to an item that will be filtered out Callers should paginate until next_page_token is empty to retrieve all accessible streams..
type MaterializedFeature ¶
type MaterializedFeature struct {
// Server-assigned unique identifier for the materialized feature.
MaterializedFeatureId *string `fieldmask:"materialized_feature_id"`
// The full name of the feature in Unity Catalog.
FeatureName *string `fieldmask:"feature_name"`
Destination isMaterializedFeature_Destination
// The fully qualified Unity Catalog path to the table containing the
// materialized feature (Delta table or Lakebase table). Output only.
TableName *string `fieldmask:"table_name"`
// The schedule state of the materialization pipeline. Hidden from GraphQL:
// being deprecated, so not exposed to Catalog Explorer.
PipelineScheduleState MaterializedFeature_PipelineScheduleState `fieldmask:"pipeline_schedule_state"`
// The timestamp when the pipeline last ran and updated the materialized feature
// values. If the pipeline has not run yet, this field will be null.
LastMaterializationTime *types.Time `fieldmask:"last_materialization_time"`
// True if this is an online materialized feature. False if it is an offline
// materialized feature.
IsOnline *bool `fieldmask:"is_online"`
// The trigger configuration for the materialization pipeline.
Trigger isMaterializedFeature_Trigger
// contains filtered or unexported fields
}
A materialized feature represents a feature that is continuously computed and stored..
type MaterializedFeature_Destination_OfflineStoreConfig ¶
type MaterializedFeature_Destination_OfflineStoreConfig struct {
OfflineStoreConfig OfflineStoreConfig `fieldmask:"offline_store_config"`
}
MaterializedFeature_Destination_OfflineStoreConfig selects OfflineStoreConfig for MaterializedFeature.Destination. Destination for writing feature values to an offline Delta table.
type MaterializedFeature_Destination_OnlineStoreConfig ¶
type MaterializedFeature_Destination_OnlineStoreConfig struct {
OnlineStoreConfig OnlineStoreConfig `fieldmask:"online_store_config"`
}
MaterializedFeature_Destination_OnlineStoreConfig selects OnlineStoreConfig for MaterializedFeature.Destination. Destination for writing feature values to an online Lakebase table.
type MaterializedFeature_PipelineScheduleState ¶
type MaterializedFeature_PipelineScheduleState string
const ( MaterializedFeature_PipelineScheduleState_Unspecified MaterializedFeature_PipelineScheduleState = "" // Pipeline was configured to run once then stop. MaterializedFeature_PipelineScheduleState_Snapshot MaterializedFeature_PipelineScheduleState = "SNAPSHOT" // Pipeline is actively running and computing features. MaterializedFeature_PipelineScheduleState_Active MaterializedFeature_PipelineScheduleState = "ACTIVE" // Pipeline is paused and not computing features. MaterializedFeature_PipelineScheduleState_Paused MaterializedFeature_PipelineScheduleState = "PAUSED" )
type MaterializedFeature_Trigger_CronScheduleTrigger ¶
type MaterializedFeature_Trigger_CronScheduleTrigger struct {
CronScheduleTrigger CronSchedule `fieldmask:"cron_schedule_trigger"`
}
MaterializedFeature_Trigger_CronScheduleTrigger selects CronScheduleTrigger for MaterializedFeature.Trigger. A cron-based schedule trigger for the materialization pipeline.
type MaterializedFeature_Trigger_StreamingMode ¶
type MaterializedFeature_Trigger_StreamingMode struct {
StreamingMode StreamingMode `fieldmask:"streaming_mode"`
}
MaterializedFeature_Trigger_StreamingMode selects StreamingMode for MaterializedFeature.Trigger. The Structured Streaming trigger mode used for materialization. Real-time mode (RTM) targets sub-second latency for operational workloads; micro-batch mode (MBM) favors cost efficiency for ETL and analytics workloads.
type MaterializedFeature_Trigger_TableTrigger ¶
type MaterializedFeature_Trigger_TableTrigger struct {
TableTrigger TableTrigger `fieldmask:"table_trigger"`
}
MaterializedFeature_Trigger_TableTrigger selects TableTrigger for MaterializedFeature.Trigger. A trigger that fires when the upstream source table changes.
type MaxFunction ¶
type MaxFunction struct {
// The input column from which the maximum is computed.
Input *string `fieldmask:"input"`
}
Computes the maximum value..
type MinFunction ¶
type MinFunction struct {
// The input column from which the minimum is computed.
Input *string `fieldmask:"input"`
}
Computes the minimum value..
type MtlsConfig ¶
type MtlsConfig struct {
// Unity Catalog volume path to the JKS keystore file containing the client
// certificate and private key. e.g.
// "/Volumes/<catalog>/<schema>/<volume>/client.jks". The materialization
// compute must have read permission on this volume.
KeystoreLocation *string `fieldmask:"keystore_location"`
// Secret-scope reference for the JKS keystore password.
KeystorePasswordRef *SecretScopeReference `fieldmask:"keystore_password_ref"`
// Secret-scope reference for the private key password. Often the same value as
// the keystore password (keytool's default), but provided as a separate field
// because Apache Kafka requires it as a distinct option
// (kafka.ssl.key.password).
KeyPasswordRef *SecretScopeReference `fieldmask:"key_password_ref"`
// Unity Catalog volume path to the JKS truststore file containing the CA
// certificate(s) trusted to verify the Kafka broker's server certificate. e.g.
// "/Volumes/<catalog>/<schema>/<volume>/truststore.jks".
TruststoreLocation *string `fieldmask:"truststore_location"`
// Secret-scope reference for the JKS truststore password.
TruststorePasswordRef *SecretScopeReference `fieldmask:"truststore_password_ref"`
// Set to true only when the broker certificate's SAN intentionally does not
// match the connection endpoint — for example when reaching the cluster
// through a PrivateLink endpoint whose DNS name is not in the broker
// certificate. Skipping the hostname check removes a defense against
// man-in-the-middle attacks; do not enable casually. mTLS client authentication
// is unaffected by this option.
//
// See the Apache Kafka SSL security guide for background on this check:
// https://kafka.apache.org/42/security/encryption-and-authentication-using-ssl/#host-name-verification
DisableHostnameVerification *bool `fieldmask:"disable_hostname_verification"`
}
Mutual-TLS (mTLS) authentication configuration. The keystore (client certificate + private key) and truststore (CAs trusted to verify the broker) live as JKS files on Unity Catalog volumes, with their passwords stored in <Databricks> secret scopes. This matches the SSL setup pattern documented at https://docs.databricks.com/en/connect/streaming/kafka/authentication#use-ssl-to-connect-databricks-to-kafka.
At materialization time, the generated PySpark code passes the JKS file paths and resolved passwords through to the Kafka SSL options (kafka.ssl.keystore.location, kafka.ssl.keystore.password, kafka.ssl.key.password, kafka.ssl.truststore.location, kafka.ssl.truststore.password). Passwords are resolved on the Spark cluster via dbutils.secrets.get; this message stores only references, never password values..
type OfflineStoreConfig ¶
type OfflineStoreConfig struct {
// The Unity Catalog catalog name.
CatalogName *string `fieldmask:"catalog_name"`
// The Unity Catalog schema name.
SchemaName *string `fieldmask:"schema_name"`
// Prefix for Unity Catalog table name. The materialized feature will be stored
// in a table with this prefix and a generated postfix.
TableNamePrefix *string `fieldmask:"table_name_prefix"`
}
Configuration for offline store destination..
type OnlineStoreConfig ¶
type OnlineStoreConfig struct {
// The Unity Catalog catalog name. This name is also used as the Lakebase
// logical database name. Quoting is handled by the backend where needed, do not
// pre-quote it.
CatalogName *string `fieldmask:"catalog_name"`
// The Unity Catalog schema name. This name is also used as the Lakebase schema
// name under the database. Quoting is handled by the backend where needed, do
// not pre-quote it.
SchemaName *string `fieldmask:"schema_name"`
// Prefix for Unity Catalog table name. The materialized feature will be stored
// in a Lakebase table with this prefix and a generated postfix.
TableNamePrefix *string `fieldmask:"table_name_prefix"`
// The name of the target online store.
OnlineStoreName *string `fieldmask:"online_store_name"`
}
Configuration for online store destination..
type ProtoSchemaSpec ¶
type ProtoSchemaSpec struct {
// The raw .proto file text (proto2 and proto3 syntax supported, see
// https://protobuf.dev/programming-guides/proto3/ and
// https://protobuf.dev/programming-guides/proto2/).
SchemaText *string `fieldmask:"schema_text"`
// The fully-qualified name of the message within schema_text that describes the
// Kafka payload (e.g. "Event" or "com.example.Event" if schema_text declares a
// package). Identifies which message is used to decode each Kafka record — a
// .proto file may declare multiple messages but only one represents the
// payload. Must not be empty.
MessageName *string `fieldmask:"message_name"`
}
A Protocol Buffer schema paired with the name of the message within it that describes the Kafka payload. A .proto file may declare multiple messages; message_name disambiguates..
type RequestSource ¶
type RequestSource struct {
// The schema describing the request-time fields. Currently only flat schemas
// are supported.
Schema isRequestSource_Schema
// contains filtered or unexported fields
}
A request-time data source whose value is provided at inference time: offline batch scoring or online serving endpoint.
type RequestSource_Schema_FlatSchema ¶
type RequestSource_Schema_FlatSchema struct {
FlatSchema FlatSchema `fieldmask:"flat_schema"`
}
RequestSource_Schema_FlatSchema selects FlatSchema for RequestSource.Schema. A flat schema with scalar-typed fields only.
type RollingWindow ¶
type RollingWindow struct {
// The duration of the rolling window. Must be positive when set; absent means
// lifetime (aggregate over the entity's entire history).
WindowDuration *types.Duration `fieldmask:"window_duration"`
// Non-negative analytic lag that evaluates the window this far in the past. Use
// this for timing variations unrelated to source lateness, such as a 30-day
// count as of one week ago. If unset, the analytic lag is zero. It composes
// with source.lateness when both are set.
Delay *types.Duration `fieldmask:"delay"`
}
A rolling time window with an optional non-negative delay..
type SawtoothWindow ¶
type SawtoothWindow struct {
// The duration of the window. Must be positive and span more than two days when
// set, so that both the batch (N-1 day) and stale-path (N-2 day) partial
// aggregates are well defined. The duration need not be a whole number of days
// (e.g. 3 days 15 minutes is allowed). Absent means lifetime (aggregate over
// the entity's entire history).
WindowDuration *types.Duration `fieldmask:"window_duration"`
// Delay is not currently supported for Sawtooth windows.
Delay *types.Duration `fieldmask:"delay"`
}
A sawtooth window served via the hybrid batch + streaming path. The batch pipeline maintains daily partial aggregates for the bulk of the window while the streaming pipeline maintains the most recent day(s), and serving merges them on read. Same field shape as RollingWindow, but a distinct type so the control plane can explicitly identify hybrid (sawtooth) features rather than inferring hybrid behavior from window_duration..
type ScalarDataType ¶
type ScalarDataType string
Scalar data types for request-time field definitions. Only flat (non-nested) types are supported.
const ( ScalarDataType_Unspecified ScalarDataType = "" ScalarDataType_Integer ScalarDataType = "INTEGER" ScalarDataType_Float ScalarDataType = "FLOAT" ScalarDataType_Boolean ScalarDataType = "BOOLEAN" ScalarDataType_String ScalarDataType = "STRING" ScalarDataType_Double ScalarDataType = "DOUBLE" ScalarDataType_Long ScalarDataType = "LONG" ScalarDataType_Timestamp ScalarDataType = "TIMESTAMP" ScalarDataType_Date ScalarDataType = "DATE" ScalarDataType_Short ScalarDataType = "SHORT" ScalarDataType_Binary ScalarDataType = "BINARY" ScalarDataType_Decimal ScalarDataType = "DECIMAL" )
type SchemaConfig ¶
type SchemaConfig struct {
Schema isSchemaConfig_Schema
// contains filtered or unexported fields
}
type SchemaConfig_Schema_AvroSchema ¶
type SchemaConfig_Schema_AvroSchema struct {
AvroSchema string `fieldmask:"avro_schema"`
}
SchemaConfig_Schema_AvroSchema selects AvroSchema for SchemaConfig.Schema. Avro schema in JSON format (https://avro.apache.org/docs/current/specification/).
type SchemaConfig_Schema_JsonSchema ¶
type SchemaConfig_Schema_JsonSchema struct {
JsonSchema string `fieldmask:"json_schema"`
}
SchemaConfig_Schema_JsonSchema selects JsonSchema for SchemaConfig.Schema. Schema of the JSON object in standard IETF JSON schema format (https://json-schema.org/).
type SchemaConfig_Schema_ProtoSchema ¶
type SchemaConfig_Schema_ProtoSchema struct {
ProtoSchema ProtoSchemaSpec `fieldmask:"proto_schema"`
}
SchemaConfig_Schema_ProtoSchema selects ProtoSchema for SchemaConfig.Schema. Protocol Buffer schema with its payload message name.
type SchemaLocator ¶
type SchemaLocator struct {
// Registry-specific schema locator.
RegistrySchema isSchemaLocator_RegistrySchema
// Serialization format for this schema.
Format SchemaLocator_Format `fieldmask:"format"`
// contains filtered or unexported fields
}
Schema locator for one side (payload or key) of a message. Identifies which schema to use in the schema registry and the serialization format..
type SchemaLocator_ConfluentSchema ¶
type SchemaLocator_ConfluentSchema struct {
// The Confluent schema registry subject name.
Subject *string `fieldmask:"subject"`
}
Confluent Schema Registry schema locator. The value to provide for `subject` depends on the naming strategy configured in your registry: - TopicNameStrategy (default): "{topic}-key" or "{topic}-value" e.g. for topic "transactions" use "transactions-value" for the payload and "transactions-key" for the key. - RecordNameStrategy: the fully-qualified record name e.g. "com.example.Payment" for Avro, the bare message name (without package) for Protobuf, or the `title` field value for JSON. - TopicRecordNameStrategy: "{topic}-{fully-qualified-record-name}" e.g. "transactions-com.example.Payment"..
type SchemaLocator_Format ¶
type SchemaLocator_Format string
Supported serialization formats for a schema registry schema.
const ( SchemaLocator_Format_Unspecified SchemaLocator_Format = "" // Avro-encoded schema. SchemaLocator_Format_FormatAvro SchemaLocator_Format = "FORMAT_AVRO" // Protobuf-encoded schema. SchemaLocator_Format_FormatProtobuf SchemaLocator_Format = "FORMAT_PROTOBUF" // JSON-encoded schema. SchemaLocator_Format_FormatJson SchemaLocator_Format = "FORMAT_JSON" )
type SchemaLocator_RegistrySchema_ConfluentSchema ¶
type SchemaLocator_RegistrySchema_ConfluentSchema struct {
ConfluentSchema SchemaLocator_ConfluentSchema `fieldmask:"confluent_schema"`
}
SchemaLocator_RegistrySchema_ConfluentSchema selects ConfluentSchema for SchemaLocator.RegistrySchema. Confluent Schema Registry schema locator.
type SchemaRegistryConfig ¶
type SchemaRegistryConfig struct {
// A Schema Registry UC Connection object.
UcConnection *string `fieldmask:"uc_connection"`
// Reference to the schema registry API secret in a <Databricks> secret scope.
ApiSecretRef *SecretScopeReference `fieldmask:"api_secret_ref"`
// Schema locator for the message payload. For Kafka this is the value. At least
// one of payload_schema_locator or key_schema_locator must be set.
PayloadSchemaLocator *SchemaLocator `fieldmask:"payload_schema_locator"`
// Schema locator for the message key. Only used for Kafka streams. At least one
// of payload_schema_locator or key_schema_locator must be set.
KeySchemaLocator *SchemaLocator `fieldmask:"key_schema_locator"`
}
Configuration for resolving a Stream's schema from an external schema registry (e.g. Confluent)..
type SecretScopeReference ¶
type SecretScopeReference struct {
// The <Databricks> secret scope name.
Scope *string `fieldmask:"scope"`
// The key within the scope.
Key *string `fieldmask:"key"`
}
Reference to an entry in a <Databricks> secret scope. The referenced value is fetched on the Spark cluster at materialization time via dbutils.secrets.get(scope, key)..
type SlidingWindow ¶
type SlidingWindow struct {
// The duration of the sliding window. Must be positive when set; absent means
// lifetime (aggregate over the entity's entire history).
WindowDuration *types.Duration `fieldmask:"window_duration"`
// The slide duration (interval by which windows advance, must be positive and
// less than duration).
SlideDuration *types.Duration `fieldmask:"slide_duration"`
// Non-negative analytic lag that evaluates the window this far in the past. Use
// this for timing variations unrelated to source lateness, such as a 30-day
// count as of one week ago. If unset, the analytic lag is zero. It composes
// with source.lateness when both are set.
Delay *types.Duration `fieldmask:"delay"`
// Non-negative phase shift from the default midnight UTC alignment. For
// example, offset=22h on a 24h slide produces boundaries at 22:00 UTC (17:00
// New York in standard time) instead of midnight UTC. If unset, the offset is
// zero. Must be shorter than slide_duration (and therefore window_duration).
Offset *types.Duration `fieldmask:"offset"`
}
type SourceLateness ¶
type SourceLateness struct {
// Non-negative time to wait after a window ends before treating its source data
// as complete. Training shifts the eligible evaluation time backwards by this
// duration so it does not join data that would still have been settling online.
// Materialization waits for the duration to elapse before publishing the
// window. If unset, source data is considered settled immediately.
SettlingDelay *types.Duration `fieldmask:"settling_delay"`
}
Configures when event-time data from this source is considered complete for a Feature..
type StddevPopFunction ¶
type StddevPopFunction struct {
// The input column from which the population standard deviation is computed.
// For Kafka sources, use dot-prefixed path notation (e.g., "value.amount"). For
// nested fields, the leaf node name is used. Colon-prefixed notation (e.g.,
// "value:amount") is supported for backwards compatibility but is deprecated;
// migrate to dot notation.
Input *string `fieldmask:"input"`
}
Computes the population standard deviation..
type StddevSampFunction ¶
type StddevSampFunction struct {
// The input column from which the sample standard deviation is computed.
Input *string `fieldmask:"input"`
}
Computes the sample standard deviation..
type Stream ¶
type Stream struct {
// Full three-part (catalog.schema.stream) name of the stream.
Name *string `fieldmask:"name"`
// User-provided description.
Description *string `fieldmask:"description"`
// Source-specific configuration. Determines the streaming platform source.
SourceConfig *StreamSourceConfig `fieldmask:"source_config"`
// Specifies how to connect and authenticate to the stream platform.
ConnectionConfig *StreamConnectionConfig `fieldmask:"connection_config"`
// Schema definitions for the stream, provided either directly on the Stream or
// resolved from an external schema registry through a UC Connection.
SchemaConfig *StreamSchemaConfig `fieldmask:"schema_config"`
// Configuration for streaming data ingestion: the managed table storing an
// offline copy of forward fill data and optional historical backfill.
IngestionConfig *IngestionConfig `fieldmask:"ingestion_config"`
// Time at which this Stream was created.
CreateTime *types.Time `fieldmask:"create_time"`
// Username of the Stream creator.
CreatedBy *string `fieldmask:"created_by"`
// Time at which this Stream was last modified.
UpdateTime *types.Time `fieldmask:"update_time"`
// Username of user who last modified the Stream.
UpdatedBy *string `fieldmask:"updated_by"`
// Indicates whether the principal is limited to retrieving metadata for the
// associated object through the BROWSE privilege when include_browse is enabled
// in the request.
BrowseOnly *bool `fieldmask:"browse_only"`
}
A Stream is a governed UC entity representing an external streaming data source. The source_config oneof determines the streaming platform source (e.g. Kafka, Kinesis, etc.)..
type StreamArnList ¶
type StreamArnList struct {
// Kinesis stream ARNs to read from. For example,
// 'arn:aws:kinesis:us-west-2:111122223333:stream/stream-a'.
Arns []string `fieldmask:"arns"`
}
A list of Kinesis stream ARNs to read from..
type StreamConnectionConfig ¶
type StreamConnectionConfig struct {
ConnectionConfig isStreamConnectionConfig_ConnectionConfig
// contains filtered or unexported fields
}
Specifies how to connect and authenticate to the stream platform..
type StreamConnectionConfig_ConnectionConfig_DirectMtlsConfig ¶
type StreamConnectionConfig_ConnectionConfig_DirectMtlsConfig struct {
DirectMtlsConfig DirectMtlsConfig `fieldmask:"direct_mtls_config"`
}
StreamConnectionConfig_ConnectionConfig_DirectMtlsConfig selects DirectMtlsConfig for StreamConnectionConfig.ConnectionConfig. Direct mTLS configuration for stream platform access. This is only used in the short term until UC Kafka Connections support mTLS . Once UC Kafka Connections support mTLS, this will be deprecated.
type StreamConnectionConfig_ConnectionConfig_UcConnectionName ¶
type StreamConnectionConfig_ConnectionConfig_UcConnectionName struct {
UcConnectionName string `fieldmask:"uc_connection_name"`
}
StreamConnectionConfig_ConnectionConfig_UcConnectionName selects UcConnectionName for StreamConnectionConfig.ConnectionConfig. Name of an existing UC Connection for stream platform access. Must be the correct type for the streaming platform (e.g. a Kafka Connection for a Kafka Stream, or a Kinesis Connection for a Kinesis Stream).
type StreamNameList ¶
type StreamNameList struct {
// Kinesis stream names to read from.
Names []string `fieldmask:"names"`
}
A list of Kinesis stream names to read from..
type StreamSchemaConfig ¶
type StreamSchemaConfig struct {
SchemaConfig isStreamSchemaConfig_SchemaConfig
// contains filtered or unexported fields
}
Schema definitions for the stream. Feature store supports both direct schemas and schema registries..
type StreamSchemaConfig_SchemaConfig_DirectSchemas ¶
type StreamSchemaConfig_SchemaConfig_DirectSchemas struct {
DirectSchemas DirectSchemas `fieldmask:"direct_schemas"`
}
StreamSchemaConfig_SchemaConfig_DirectSchemas selects DirectSchemas for StreamSchemaConfig.SchemaConfig. Schema definitions provided directly on the Stream.
type StreamSchemaConfig_SchemaConfig_SchemaRegistryConfig ¶
type StreamSchemaConfig_SchemaConfig_SchemaRegistryConfig struct {
SchemaRegistryConfig SchemaRegistryConfig `fieldmask:"schema_registry_config"`
}
StreamSchemaConfig_SchemaConfig_SchemaRegistryConfig selects SchemaRegistryConfig for StreamSchemaConfig.SchemaConfig. Resolve schemas from an external schema registry.
type StreamSource ¶
type StreamSource struct {
// Three-part full name of the Stream (catalog.schema.stream).
FullName *string `fieldmask:"full_name"`
// The filter condition applied to the source data before aggregation.
FilterCondition *string `fieldmask:"filter_condition"`
// The pipeline runs these SQL statements immediately after conversion into the
// schema specified on the Stream object.
TransformationSql *string `fieldmask:"transformation_sql"`
// Schema of the resulting dataframe after transformations, in Spark StructType
// JSON format (from df.schema.json()). Any subsequent functions operate against
// this dataframe.
DataframeSchema *string `fieldmask:"dataframe_schema"`
}
A Stream entity used as a data source for a feature..
type StreamSourceConfig ¶
type StreamSourceConfig struct {
SourceConfig isStreamSourceConfig_SourceConfig
// contains filtered or unexported fields
}
Source-specific configuration. Determines the streaming platform source..
type StreamSourceConfig_SourceConfig_KafkaStreamConfig ¶
type StreamSourceConfig_SourceConfig_KafkaStreamConfig struct {
KafkaStreamConfig KafkaStreamConfig `fieldmask:"kafka_stream_config"`
}
StreamSourceConfig_SourceConfig_KafkaStreamConfig selects KafkaStreamConfig for StreamSourceConfig.SourceConfig. Configuration for Apache Kafka streams.
type StreamSourceConfig_SourceConfig_KinesisStreamConfig ¶
type StreamSourceConfig_SourceConfig_KinesisStreamConfig struct {
KinesisStreamConfig KinesisStreamConfig `fieldmask:"kinesis_stream_config"`
}
StreamSourceConfig_SourceConfig_KinesisStreamConfig selects KinesisStreamConfig for StreamSourceConfig.SourceConfig. Configuration for AWS Kinesis Data Streams.
type StreamingMode ¶
type StreamingMode struct {
// The type of streaming mode used by the materialization pipeline.
Mode StreamingMode_StreamingModeType `fieldmask:"mode"`
// The desired data freshness for feature materialization, expressed as a
// duration string (e.g. "1 minute").
FreshnessTarget *string `fieldmask:"freshness_target"`
}
The streaming mode configuration for a streaming materialization pipeline..
type StreamingMode_StreamingModeType ¶
type StreamingMode_StreamingModeType string
const ( StreamingMode_StreamingModeType_Unspecified StreamingMode_StreamingModeType = "" // Real-time mode. Ultra-low-latency trigger intended for operational workloads // that need responses in milliseconds or sub-second latency. StreamingMode_StreamingModeType_StreamingModeTypeRtm StreamingMode_StreamingModeType = "STREAMING_MODE_TYPE_RTM" // Micro-batch mode in Structured Streaming. Better suited for ETL and analytics // workloads where latency is measured in seconds or minutes and cost efficiency // matters more. StreamingMode_StreamingModeType_StreamingModeTypeMbm StreamingMode_StreamingModeType = "STREAMING_MODE_TYPE_MBM" )
type SubscriptionMode
deprecated
type SubscriptionMode struct {
// These match the settings from
// https://spark.apache.org/docs/latest/streaming/structured-streaming-kafka-integration.html
SubscriptionMode isSubscriptionMode_SubscriptionMode
// contains filtered or unexported fields
}
Deprecated: Use KafkaSubscriptionMode instead..
type SubscriptionMode_SubscriptionMode_Assign ¶
type SubscriptionMode_SubscriptionMode_Assign struct {
Assign string `fieldmask:"assign"`
}
SubscriptionMode_SubscriptionMode_Assign selects Assign for SubscriptionMode.SubscriptionMode. A JSON string that contains the specific topic-partitions to consume from. For example, for '{"topicA":[0,1],"topicB":[2,4]}', topicA's 0'th and 1st partitions will be consumed from.
type SubscriptionMode_SubscriptionMode_Subscribe ¶
type SubscriptionMode_SubscriptionMode_Subscribe struct {
Subscribe string `fieldmask:"subscribe"`
}
SubscriptionMode_SubscriptionMode_Subscribe selects Subscribe for SubscriptionMode.SubscriptionMode. A comma-separated list of Kafka topics to read from. For example, 'topicA,topicB,topicC'.
type SubscriptionMode_SubscriptionMode_SubscribePattern ¶
type SubscriptionMode_SubscriptionMode_SubscribePattern struct {
SubscribePattern string `fieldmask:"subscribe_pattern"`
}
SubscriptionMode_SubscriptionMode_SubscribePattern selects SubscribePattern for SubscriptionMode.SubscriptionMode. A regular expression matching topics to subscribe to. For example, 'topic.*' will subscribe to all topics starting with 'topic'.
type SumFunction ¶
type SumFunction struct {
// The input column from which the sum is computed. For Kafka sources, use
// dot-prefixed path notation (e.g., "value.amount"). For nested fields, the
// leaf node name is used. Colon-prefixed notation (e.g., "value:amount") is
// supported for backwards compatibility but is deprecated; migrate to dot
// notation.
Input *string `fieldmask:"input"`
}
Computes the sum of values..
type TableTrigger ¶
type TableTrigger struct {
}
A trigger that fires when the upstream source table changes..
type TimeWindow ¶
type TimeWindow struct {
WindowType isTimeWindow_WindowType
// Earliest event-time boundary at which the Feature may emit an output. This
// gates outputs, not the historical inputs read by a window. For example, a
// 365-day window with start_time=2026-01-01 begins emitting partial-window
// values on that date instead of waiting for 365 days of data; a lifetime
// window produces no output before start_time. If unset, tumbling and
// fixed-duration sliding windows first emit at an offset-aligned boundary after
// a full window can be formed. If unset, lifetime sliding windows and rolling
// windows emit as soon as eligible source data exists.
StartTime *types.Time `fieldmask:"start_time"`
// contains filtered or unexported fields
}
type TimeWindow_WindowType_Rolling ¶
type TimeWindow_WindowType_Rolling struct {
Rolling RollingWindow `fieldmask:"rolling"`
}
TimeWindow_WindowType_Rolling selects Rolling for TimeWindow.WindowType.
type TimeWindow_WindowType_Sawtooth ¶
type TimeWindow_WindowType_Sawtooth struct {
Sawtooth SawtoothWindow `fieldmask:"sawtooth"`
}
TimeWindow_WindowType_Sawtooth selects Sawtooth for TimeWindow.WindowType. A sawtooth window served via the hybrid batch + streaming path.
type TimeWindow_WindowType_Sliding ¶
type TimeWindow_WindowType_Sliding struct {
Sliding SlidingWindow `fieldmask:"sliding"`
}
TimeWindow_WindowType_Sliding selects Sliding for TimeWindow.WindowType.
type TimeWindow_WindowType_Tumbling ¶
type TimeWindow_WindowType_Tumbling struct {
Tumbling TumblingWindow `fieldmask:"tumbling"`
}
TimeWindow_WindowType_Tumbling selects Tumbling for TimeWindow.WindowType.
type TimeseriesColumn ¶
type TimeseriesColumn struct {
// The name of the timeseries column. For Kafka sources, use dot-prefixed path
// notation to reference fields within the key or value schema (e.g.,
// "value.event_timestamp"). For nested fields, the leaf node name (e.g.,
// "event_timestamp" from "value.event_details.event_timestamp") is what will be
// present in materialized tables and expected to match at query time.
// Colon-prefixed notation (e.g., "value:event_timestamp") is supported for
// backwards compatibility but is deprecated; migrate to dot notation.
Name *string `fieldmask:"name"`
}
type TumblingWindow ¶
type TumblingWindow struct {
// The duration of each tumbling window (non-overlapping, fixed-duration
// windows).
WindowDuration *types.Duration `fieldmask:"window_duration"`
// Non-negative analytic lag that evaluates the window this far in the past. Use
// this for timing variations unrelated to source lateness, such as a 30-day
// count as of one week ago. If unset, the analytic lag is zero. It composes
// with source.lateness when both are set.
Delay *types.Duration `fieldmask:"delay"`
// Non-negative phase shift from the default midnight UTC alignment. For
// example, offset=22h on a 24h window produces boundaries at 22:00 UTC (17:00
// New York in standard time) instead of midnight UTC. If unset, the offset is
// zero. Must be shorter than window_duration.
Offset *types.Duration `fieldmask:"offset"`
}
type UpdateFeatureRequest ¶
type UpdateKafkaConfigRequest ¶
type UpdateKafkaConfigRequest struct {
// The Kafka config to update.
KafkaConfig *KafkaConfig
// The list of fields to update.
UpdateMask *types.FieldMask[KafkaConfig]
}
type UpdateMaterializedFeatureRequest ¶
type UpdateMaterializedFeatureRequest struct {
// The materialized feature to update.
MaterializedFeature *MaterializedFeature
// Provide the materialization feature fields which should be updated.
// Currently, only the pipeline_state field can be updated.
UpdateMask *types.FieldMask[MaterializedFeature]
}
type UpdateStreamRequest ¶
type UpdateStreamRequest struct {
// The Stream to update.
Stream *Stream
// The list of fields to update.
UpdateMask *types.FieldMask[Stream]
}
Update a Stream. Only fields listed in `update_mask` are mutated..
type VarPopFunction ¶
type VarPopFunction struct {
// The input column from which the population variance is computed.
Input *string `fieldmask:"input"`
}
Computes the population variance..
type VarSampFunction ¶
type VarSampFunction struct {
// The input column from which the sample variance is computed.
Input *string `fieldmask:"input"`
}
Computes the sample variance..