From 1802ac48166a4af795e6631d64f07b1504dcf592 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 12:22:52 -0700 Subject: [PATCH 01/19] Fix Signed-off-by: Kevin Zhang --- docs/getting-started/concepts/feature-view.md | 8 +- .../adding-a-new-offline-store.md | 2 +- .../java-demo/feature_repo/driver_repo.py | 12 ++- .../docker-compose/feast10/definitions.py | 8 +- protos/feast/core/DataSource.proto | 6 +- sdk/python/feast/data_source.py | 94 ++++++++++++++++--- sdk/python/feast/field.py | 3 + sdk/python/feast/types.py | 6 ++ .../feature_repos/universal/feature_views.py | 10 +- .../registration/test_inference.py | 13 ++- .../integration/registration/test_registry.py | 8 +- sdk/python/tests/unit/test_data_sources.py | 24 ++++- ui/feature_repo/features.py | 7 +- 13 files changed, 168 insertions(+), 33 deletions(-) diff --git a/docs/getting-started/concepts/feature-view.md b/docs/getting-started/concepts/feature-view.md index fa6612fbadd..2786591d29e 100644 --- a/docs/getting-started/concepts/feature-view.md +++ b/docs/getting-started/concepts/feature-view.md @@ -144,10 +144,10 @@ from feast import Field, Float64, RequestSource # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( name="vals_to_add", - schema={ - "val_to_add": ValueType.INT64, - "val_to_add_2": ValueType.INT64 - } + schema=[ + Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add_2": dtype=PrimitiveFeastType.INT64), + ] ) # Use the input data and feature view features to create new features diff --git a/docs/how-to-guides/adding-a-new-offline-store.md b/docs/how-to-guides/adding-a-new-offline-store.md index 27b70421c37..ee2a8ff9316 100644 --- a/docs/how-to-guides/adding-a-new-offline-store.md +++ b/docs/how-to-guides/adding-a-new-offline-store.md @@ -246,7 +246,7 @@ Finally, the custom data source class can be use in the feature repo to define a ```python pdriver_hourly_stats = CustomFileDataSource( path="feature_repo/data/driver_stats.parquet", - event_timestamp_column="event_timestamp", + timestamp_field="event_timestamp", created_timestamp_column="created", ) diff --git a/examples/java-demo/feature_repo/driver_repo.py b/examples/java-demo/feature_repo/driver_repo.py index d7ce41134a7..78280eb08d5 100644 --- a/examples/java-demo/feature_repo/driver_repo.py +++ b/examples/java-demo/feature_repo/driver_repo.py @@ -5,6 +5,8 @@ from feast.request_feature_view import RequestFeatureView from feast.types import Float32, Float64, Int64, String from google.protobuf.duration_pb2 import Duration +from feast.field import Field +from feast.types import PrimitiveFeastType from feast import Entity, Feature, FeatureView, FileSource, ValueType @@ -33,7 +35,10 @@ # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( name="vals_to_add", - schema={"val_to_add": ValueType.INT64, "val_to_add_2": ValueType.INT64}, + schema=[ + Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add_2", dtype=PrimitiveFeastType.INT64), + ], ) # Define an on demand feature view which can generate new features based on @@ -59,6 +64,9 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_age_request_fv = RequestFeatureView( name="driver_age", request_data_source=RequestSource( - name="driver_age", schema={"driver_age": ValueType.INT64,} + name="driver_age", + schema=[ + Field(name="driver_age", dtype=PrimitiveFeastType.INT64), + ], ), ) diff --git a/java/serving/src/test/resources/docker-compose/feast10/definitions.py b/java/serving/src/test/resources/docker-compose/feast10/definitions.py index 35c5a2c127f..8cfea19e0db 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/definitions.py +++ b/java/serving/src/test/resources/docker-compose/feast10/definitions.py @@ -5,9 +5,10 @@ from feast.feature_view import FeatureView from feast.field import Field from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Float64, Int64 +from feast.types import Float32, Float64, Int64, PrimitiveFeastType from feast.value_type import ValueType from google.protobuf.duration_pb2 import Duration +from feast import FileSource from feast import FileSource @@ -42,7 +43,10 @@ input_request = RequestSource( name="vals_to_add", - schema={"val_to_add": ValueType.INT64, "val_to_add_2": ValueType.INT64}, + schema=[ + Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add_2", dtype=PrimitiveFeastType.INT64), + ], ) diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 5bd91e5703e..d958281ca2c 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -24,6 +24,7 @@ option java_package = "feast.proto.core"; import "feast/core/DataFormat.proto"; import "feast/types/Value.proto"; +import "feast/core/Feature.proto"; // Defines a Data Source that can be used source Feature data // Next available id: 28 @@ -212,7 +213,10 @@ message DataSource { message RequestDataOptions { reserved 1; // Mapping of feature name to type - map schema = 2; + map deprecated_schema = 2; + + repeated FeatureSpecV2 schema = 3; + } // Defines options for DataSource that supports pushing data to it. This allows data to be pushed to diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index db7764493cb..393900422dc 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -13,9 +13,10 @@ # limitations under the License. import enum +import re import warnings from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Iterable, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union, List from google.protobuf.json_format import MessageToJson @@ -24,6 +25,9 @@ from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.repo_config import RepoConfig, get_data_source_class_from_type from feast.value_type import ValueType +from numpy import deprecate +from feast.field import Field +from feast.types import VALUE_TYPES_TO_FEAST_TYPES class SourceType(enum.Enum): @@ -449,7 +453,7 @@ class RequestSource(DataSource): Args: name: Name of the request data source - schema: Schema mapping from the input feature name to a ValueType + schema Union[Dict[str, ValueType], List[Field]]: Schema mapping from the input feature name to a ValueType description (optional): A human-readable description. tags (optional): A dictionary of key-value pairs to store arbitrary metadata. owner (optional): The owner of the request data source, typically the email of the primary @@ -457,18 +461,24 @@ class RequestSource(DataSource): """ name: str - schema: Dict[str, ValueType] + schema: Union[Dict[str, ValueType], List[Field]] def __init__( self, name: str, - schema: Dict[str, ValueType], + schema: Union[Dict[str, ValueType], List[Field]], description: Optional[str] = "", tags: Optional[Dict[str, str]] = None, owner: Optional[str] = "", ): """Creates a RequestSource object.""" super().__init__(name=name, description=description, tags=tags, owner=owner) + if isinstance(schema, Dict): + warnings.warn( + "Schema in RequestSource is changing type. The schema data type Dict[str, ValueType] is being deprecated in Feast 0.23. " + "Please use List[Field] instead for the schema", + DeprecationWarning, + ) self.schema = schema def validate(self, config: RepoConfig): @@ -479,12 +489,67 @@ def get_table_column_names_and_types( ) -> Iterable[Tuple[str, str]]: pass + def __eq__(self, other): + if not isinstance(other, RequestSource): + raise TypeError( + "Comparisons should only involve RequestSource class objects." + ) + if ( + self.name != other.name or + self.description != other.description or + self.owner != other.owner or + self.tags != other.tags + ): + return False + else: + if isinstance(self.schema, List) and isinstance(other.schema, List): + for field1, field2 in zip(self.schema, other.schema): + if field1 != field2: + return False + return True + elif isinstance(self.schema, Dict) and isinstance(other.schema, Dict): + for key, value in self.schema.items(): + if key not in other.schema: + return False + elif value != other.schema[key]: + return False + return True + elif isinstance(self.schema, Dict) and isinstance(other.schema, List): + dict_schema = self.schema + list_schema = other.schema + elif isinstance(self.schema, List) and isinstance(other.schema, Dict): + dict_schema = other.schema + list_schema = self.schema + + temp_schema = {} + for field in list_schema: + temp_schema[field.name] = field.dtype + for name, value in dict_schema.items(): + if name not in temp_schema: + return False + elif VALUE_TYPES_TO_FEAST_TYPES[value.value] != temp_schema[name]: + return False + return True + @staticmethod def from_proto(data_source: DataSourceProto): + + deprecated_schema = data_source.request_data_options.deprecated_schema schema_pb = data_source.request_data_options.schema - schema = {} - for key, val in schema_pb.items(): - schema[key] = ValueType(val) + + schema = [] + if deprecated_schema and not schema_pb: + warnings.warn( + "Schema in RequestSource is changing type. The schema data type Dict[str, ValueType] is being deprecated in Feast 0.23. " + "Please use List[Field] instead for the schema", + DeprecationWarning, + ) + for key, val in deprecated_schema.items(): + schema[key] = ValueType(val) + else: + for field_proto in schema_pb: + schema.append(Field.from_proto(field_proto)) + return RequestSource( name=data_source.name, schema=schema, @@ -494,18 +559,23 @@ def from_proto(data_source: DataSourceProto): ) def to_proto(self) -> DataSourceProto: - schema_pb = {} - for key, value in self.schema.items(): - schema_pb[key] = value.value - options = DataSourceProto.RequestDataOptions(schema=schema_pb) + + schema_pb = [] + + if isinstance(self.schema, Dict): + for key, value in self.schema.items(): + schema_pb.append(Field(name=key, dtype=VALUE_TYPES_TO_FEAST_TYPES[value.value]).to_proto()) + else: + for field in self.schema: + schema_pb.append(field.to_proto()) data_source_proto = DataSourceProto( name=self.name, type=DataSourceProto.REQUEST_SOURCE, - request_data_options=options, description=self.description, tags=self.tags, owner=self.owner, ) + data_source_proto.request_data_options.schema.extend(schema_pb) return data_source_proto diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 2198d8a3f1b..90c2f933aab 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from feast.protos.feast.types.Value_pb2 import ValueType from feast.feature import Feature from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FieldProto from feast.types import FeastType, from_value_type @@ -62,6 +63,8 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" + if isinstance(self.dtype, ValueType): + return self.dtype.Enum value_type = self.dtype.to_value_type() return FieldProto(name=self.name, value_type=value_type.value) diff --git a/sdk/python/feast/types.py b/sdk/python/feast/types.py index 29a49a72c08..9ce5840b347 100644 --- a/sdk/python/feast/types.py +++ b/sdk/python/feast/types.py @@ -80,6 +80,12 @@ def to_value_type(self) -> ValueType: def __str__(self): return PRIMITIVE_FEAST_TYPES_TO_STRING[self.name] + def __eq__(self, other): + return self.value == other.value + + def __hash__(self): + return hash((PRIMITIVE_FEAST_TYPES_TO_STRING[self.name])) + Invalid = PrimitiveFeastType.INVALID Bytes = PrimitiveFeastType.BYTES diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 8a89a81128f..e6900e2d9ef 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,9 +17,9 @@ ValueType, ) from feast.data_source import DataSource, RequestSource -from feast.types import FeastType +from feast.types import FeastType, PrimitiveFeastType from tests.integration.feature_repos.universal.entities import location - +from feast.field import Field def driver_feature_view( data_source: DataSource, @@ -123,7 +123,11 @@ def similarity_feature_view( def create_conv_rate_request_source(): - return RequestSource(name="conv_rate_input", schema={"val_to_add": ValueType.INT32}) + return RequestSource( + name="conv_rate_input", + schema=[ + Field(name="val_to_add", dtype=PrimitiveFeastType.INT32), + ]) def create_similarity_request_source(): diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index c0358abb59e..80fc60c5677 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -36,6 +36,9 @@ simple_bq_source_using_table_arg, ) +from feast.field import Field +from feast.types import PrimitiveFeastType + def test_update_entities_with_inferred_types_from_feature_views( simple_dataset_1, simple_dataset_2 @@ -168,7 +171,10 @@ def test_update_data_sources_with_inferred_event_timestamp_col(universal_data_so def test_on_demand_features_type_inference(): # Create Feature Views date_request = RequestSource( - name="date_request", schema={"some_date": ValueType.UNIX_TIMESTAMP} + name="date_request", + schema=[ + Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP), + ], ) @on_demand_feature_view( @@ -227,7 +233,10 @@ def test_view_with_missing_feature(features_df: pd.DataFrame) -> pd.DataFrame: def test_datasource_inference(): # Create Feature Views date_request = RequestSource( - name="date_request", schema={"some_date": ValueType.UNIX_TIMESTAMP} + name="date_request", + schema=[ + Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP), + ], ) @on_demand_feature_view( diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 25b0e7714ec..12da0816922 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -31,7 +31,8 @@ from feast.repo_config import RegistryConfig from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType - +from feast.field import Field +from feast.types import PrimitiveFeastType @pytest.fixture def local_registry() -> Registry: @@ -247,7 +248,10 @@ def test_modify_feature_views_success(test_registry): ) request_source = RequestSource( - name="request_source", schema={"my_input_1": ValueType.INT32} + name="request_source", + schema=[ + Field(name="my_input_1", dtype=PrimitiveFeastType.INT32), + ], ) fv1 = FeatureView( diff --git a/sdk/python/tests/unit/test_data_sources.py b/sdk/python/tests/unit/test_data_sources.py index f32089b3b9e..1dfff1afc8c 100644 --- a/sdk/python/tests/unit/test_data_sources.py +++ b/sdk/python/tests/unit/test_data_sources.py @@ -1,9 +1,12 @@ +from aiohttp import request import pytest from feast import ValueType from feast.data_source import PushSource, RequestDataSource, RequestSource from feast.infra.offline_stores.bigquery_source import BigQuerySource - +from feast.field import Field +from feast.types import PrimitiveFeastType +from sdk.python.feast.types import ComplexFeastType def test_push_with_batch(): push_source = PushSource( @@ -30,5 +33,20 @@ def test_request_data_source_deprecation(): ) request_data_source_proto = request_data_source.to_proto() returned_request_source = RequestSource.from_proto(request_data_source_proto) - assert returned_request_source.name == request_data_source.name - assert returned_request_source.schema == request_data_source.schema + assert returned_request_source == request_data_source + +def test_request_source_primitive_type_to_proto(): + schema = [ + Field(name="f1", dtype=PrimitiveFeastType.FLOAT32), + Field(name="f2", dtype=PrimitiveFeastType.BOOL), + ] + request_source = RequestSource( + name="source", + schema=schema, + description="desc", + tags={}, + owner="feast", + ) + request_proto = request_source.to_proto() + deserialized_request_source = RequestSource.from_proto(request_proto) + assert deserialized_request_source == request_source diff --git a/ui/feature_repo/features.py b/ui/feature_repo/features.py index dfc23cc394b..0cdf224b8ff 100644 --- a/ui/feature_repo/features.py +++ b/ui/feature_repo/features.py @@ -14,6 +14,8 @@ from feast.data_source import RequestSource from feast.request_feature_view import RequestFeatureView from feast.on_demand_feature_view import on_demand_feature_view +from feast.field import Field +from feast.types import PrimitiveFeastType import pandas as pd zipcode = Entity( @@ -130,7 +132,10 @@ # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( - name="transaction", schema={"transaction_amt": ValueType.INT64}, + name="transaction", + schema=[ + Field(name="transaction_amt", dtype=PrimitiveFeastType.INT64), + ], ) # Define an on demand feature view which can generate new features based on From 5fd414dc76092956196d5a1c744072b1cd0a51fa Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 12:40:01 -0700 Subject: [PATCH 02/19] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/data_source.py | 54 +++++++++++-------- sdk/python/feast/field.py | 2 +- sdk/python/feast/on_demand_feature_view.py | 8 ++- sdk/python/feast/request_feature_view.py | 16 ++++-- .../registration/test_inference.py | 13 ++--- .../integration/registration/test_registry.py | 8 ++- sdk/python/tests/unit/test_data_sources.py | 14 +++-- 7 files changed, 65 insertions(+), 50 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 393900422dc..b67e763f872 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -16,18 +16,18 @@ import re import warnings from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union, List +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union from google.protobuf.json_format import MessageToJson +from numpy import deprecate from feast import type_map from feast.data_format import StreamFormat +from feast.field import Field from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.repo_config import RepoConfig, get_data_source_class_from_type -from feast.value_type import ValueType -from numpy import deprecate -from feast.field import Field from feast.types import VALUE_TYPES_TO_FEAST_TYPES +from feast.value_type import ValueType class SourceType(enum.Enum): @@ -495,10 +495,10 @@ def __eq__(self, other): "Comparisons should only involve RequestSource class objects." ) if ( - self.name != other.name or - self.description != other.description or - self.owner != other.owner or - self.tags != other.tags + self.name != other.name + or self.description != other.description + or self.owner != other.owner + or self.tags != other.tags ): return False else: @@ -517,7 +517,7 @@ def __eq__(self, other): elif isinstance(self.schema, Dict) and isinstance(other.schema, List): dict_schema = self.schema list_schema = other.schema - elif isinstance(self.schema, List) and isinstance(other.schema, Dict): + elif isinstance(self.schema, List) and isinstance(other.schema, Dict): dict_schema = other.schema list_schema = self.schema @@ -537,26 +537,34 @@ def from_proto(data_source: DataSourceProto): deprecated_schema = data_source.request_data_options.deprecated_schema schema_pb = data_source.request_data_options.schema - schema = [] if deprecated_schema and not schema_pb: warnings.warn( "Schema in RequestSource is changing type. The schema data type Dict[str, ValueType] is being deprecated in Feast 0.23. " "Please use List[Field] instead for the schema", DeprecationWarning, ) + dict_schema = {} for key, val in deprecated_schema.items(): - schema[key] = ValueType(val) + dict_schema[key] = ValueType(val) + return RequestSource( + name=data_source.name, + schema=dict_schema, + description=data_source.description, + tags=dict(data_source.tags), + owner=data_source.owner, + ) else: + list_schema = [] for field_proto in schema_pb: - schema.append(Field.from_proto(field_proto)) - - return RequestSource( - name=data_source.name, - schema=schema, - description=data_source.description, - tags=dict(data_source.tags), - owner=data_source.owner, - ) + list_schema.append(Field.from_proto(field_proto)) + + return RequestSource( + name=data_source.name, + schema=list_schema, + description=data_source.description, + tags=dict(data_source.tags), + owner=data_source.owner, + ) def to_proto(self) -> DataSourceProto: @@ -564,7 +572,11 @@ def to_proto(self) -> DataSourceProto: if isinstance(self.schema, Dict): for key, value in self.schema.items(): - schema_pb.append(Field(name=key, dtype=VALUE_TYPES_TO_FEAST_TYPES[value.value]).to_proto()) + schema_pb.append( + Field( + name=key, dtype=VALUE_TYPES_TO_FEAST_TYPES[value.value] + ).to_proto() + ) else: for field in self.schema: schema_pb.append(field.to_proto()) diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 90c2f933aab..2c553401c0c 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from feast.protos.feast.types.Value_pb2 import ValueType from feast.feature import Feature from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FieldProto +from feast.protos.feast.types.Value_pb2 import ValueType from feast.types import FeastType, from_value_type from feast.value_type import ValueType diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index a453cbb4db7..feacab17715 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -352,7 +352,13 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): def get_request_data_schema(self) -> Dict[str, ValueType]: schema: Dict[str, ValueType] = {} for request_source in self.source_request_sources.values(): - schema.update(request_source.schema) + if isinstance(request_source.schema, List): + new_schema = {} + for field in request_source.schema: + new_schema[field.name] = field.dtype.to_value_type() + schema.update(new_schema) + elif isinstance(request_source.schema, Dict): + schema.update(request_source.schema) return schema def get_transformed_features_df( diff --git a/sdk/python/feast/request_feature_view.py b/sdk/python/feast/request_feature_view.py index 863a4b49648..c7ffbbe2cea 100644 --- a/sdk/python/feast/request_feature_view.py +++ b/sdk/python/feast/request_feature_view.py @@ -63,12 +63,20 @@ def __init__( DeprecationWarning, ) + if isinstance(request_data_source.schema, Dict): + new_features = [ + Field(name=name, dtype=dtype) + for name, dtype in request_data_source.schema.items() + ] + else: + new_features = [ + Field(name=field.name, dtype=field.dtype.to_value_type()) + for field in request_data_source.schema + ] + super().__init__( name=name, - features=[ - Field(name=name, dtype=from_value_type(value_type)) - for name, value_type in request_data_source.schema.items() - ], + features=new_features, description=description, tags=tags, owner=owner, diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index 80fc60c5677..ce3b38672ca 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -29,16 +29,13 @@ SparkSource, ) from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import String, UnixTimestamp +from feast.types import String, UnixTimestamp, PrimitiveFeastType from tests.utils.data_source_utils import ( prep_file_source, simple_bq_source_using_query_arg, simple_bq_source_using_table_arg, ) -from feast.field import Field -from feast.types import PrimitiveFeastType - def test_update_entities_with_inferred_types_from_feature_views( simple_dataset_1, simple_dataset_2 @@ -172,9 +169,7 @@ def test_on_demand_features_type_inference(): # Create Feature Views date_request = RequestSource( name="date_request", - schema=[ - Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP), - ], + schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP),], ) @on_demand_feature_view( @@ -234,9 +229,7 @@ def test_datasource_inference(): # Create Feature Views date_request = RequestSource( name="date_request", - schema=[ - Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP), - ], + schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP),], ) @on_demand_feature_view( diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 12da0816922..aa3a806a948 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -29,10 +29,10 @@ from feast.protos.feast.types import Value_pb2 as ValueProto from feast.registry import Registry from feast.repo_config import RegistryConfig -from feast.types import Array, Bytes, Float32, Int32, Int64, String +from feast.types import Array, Bytes, Float32, Int32, Int64, String, PrimitiveFeastType from feast.value_type import ValueType from feast.field import Field -from feast.types import PrimitiveFeastType + @pytest.fixture def local_registry() -> Registry: @@ -249,9 +249,7 @@ def test_modify_feature_views_success(test_registry): request_source = RequestSource( name="request_source", - schema=[ - Field(name="my_input_1", dtype=PrimitiveFeastType.INT32), - ], + schema=[Field(name="my_input_1", dtype=PrimitiveFeastType.INT32),], ) fv1 = FeatureView( diff --git a/sdk/python/tests/unit/test_data_sources.py b/sdk/python/tests/unit/test_data_sources.py index 1dfff1afc8c..f5b8ac16923 100644 --- a/sdk/python/tests/unit/test_data_sources.py +++ b/sdk/python/tests/unit/test_data_sources.py @@ -1,12 +1,13 @@ -from aiohttp import request import pytest +from aiohttp import request +from sdk.python.feast.types import ComplexFeastType from feast import ValueType from feast.data_source import PushSource, RequestDataSource, RequestSource -from feast.infra.offline_stores.bigquery_source import BigQuerySource from feast.field import Field +from feast.infra.offline_stores.bigquery_source import BigQuerySource from feast.types import PrimitiveFeastType -from sdk.python.feast.types import ComplexFeastType + def test_push_with_batch(): push_source = PushSource( @@ -35,17 +36,14 @@ def test_request_data_source_deprecation(): returned_request_source = RequestSource.from_proto(request_data_source_proto) assert returned_request_source == request_data_source + def test_request_source_primitive_type_to_proto(): schema = [ Field(name="f1", dtype=PrimitiveFeastType.FLOAT32), Field(name="f2", dtype=PrimitiveFeastType.BOOL), ] request_source = RequestSource( - name="source", - schema=schema, - description="desc", - tags={}, - owner="feast", + name="source", schema=schema, description="desc", tags={}, owner="feast", ) request_proto = request_source.to_proto() deserialized_request_source = RequestSource.from_proto(request_proto) From 9f677c785e057dc2fb2b8f006b3d4cfc3d0d23b8 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 12:42:52 -0700 Subject: [PATCH 03/19] fix lint Signed-off-by: Kevin Zhang --- sdk/python/feast/data_source.py | 2 -- sdk/python/feast/on_demand_feature_view.py | 2 +- sdk/python/tests/integration/registration/test_inference.py | 4 ++-- sdk/python/tests/integration/registration/test_registry.py | 2 +- sdk/python/tests/unit/test_data_sources.py | 2 -- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index b67e763f872..bb0d2ef7507 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -13,13 +13,11 @@ # limitations under the License. import enum -import re import warnings from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union from google.protobuf.json_format import MessageToJson -from numpy import deprecate from feast import type_map from feast.data_format import StreamFormat diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index feacab17715..ab2114cd43e 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -366,7 +366,7 @@ def get_transformed_features_df( ) -> pd.DataFrame: # Apply on demand transformations columns_to_cleanup = [] - for source_fv_projection in self.source_feature_view_projections.values(): + for source_fv_projection in self.source_feature_view_projections.values(): for feature in source_fv_projection.features: full_feature_ref = f"{source_fv_projection.name}__{feature.name}" if full_feature_ref in df_with_features.keys(): diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index ce3b38672ca..bd9ce1695b1 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -169,7 +169,7 @@ def test_on_demand_features_type_inference(): # Create Feature Views date_request = RequestSource( name="date_request", - schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP),], + schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP)], ) @on_demand_feature_view( @@ -229,7 +229,7 @@ def test_datasource_inference(): # Create Feature Views date_request = RequestSource( name="date_request", - schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP),], + schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP)], ) @on_demand_feature_view( diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index aa3a806a948..aa90deed81d 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -249,7 +249,7 @@ def test_modify_feature_views_success(test_registry): request_source = RequestSource( name="request_source", - schema=[Field(name="my_input_1", dtype=PrimitiveFeastType.INT32),], + schema=[Field(name="my_input_1", dtype=PrimitiveFeastType.INT32)], ) fv1 = FeatureView( diff --git a/sdk/python/tests/unit/test_data_sources.py b/sdk/python/tests/unit/test_data_sources.py index f5b8ac16923..49015e6ad19 100644 --- a/sdk/python/tests/unit/test_data_sources.py +++ b/sdk/python/tests/unit/test_data_sources.py @@ -1,6 +1,4 @@ import pytest -from aiohttp import request -from sdk.python.feast.types import ComplexFeastType from feast import ValueType from feast.data_source import PushSource, RequestDataSource, RequestSource From 2122172b6bcc4c6522821039fbbe5a829110c1b2 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 12:43:37 -0700 Subject: [PATCH 04/19] indent fix Signed-off-by: Kevin Zhang --- sdk/python/feast/on_demand_feature_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index ab2114cd43e..feacab17715 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -366,7 +366,7 @@ def get_transformed_features_df( ) -> pd.DataFrame: # Apply on demand transformations columns_to_cleanup = [] - for source_fv_projection in self.source_feature_view_projections.values(): + for source_fv_projection in self.source_feature_view_projections.values(): for feature in source_fv_projection.features: full_feature_ref = f"{source_fv_projection.name}__{feature.name}" if full_feature_ref in df_with_features.keys(): From 53f3439231070170a32eb11af0455bb0704c2e25 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 10:53:36 -0700 Subject: [PATCH 05/19] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/data_source.py | 48 ++++++++----------- sdk/python/feast/field.py | 1 - sdk/python/feast/on_demand_feature_view.py | 5 ++ .../feature_repos/universal/feature_views.py | 1 + 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index bb0d2ef7507..5cb941106ae 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -459,7 +459,7 @@ class RequestSource(DataSource): """ name: str - schema: Union[Dict[str, ValueType], List[Field]] + schema: List[Field] def __init__( self, @@ -477,7 +477,19 @@ def __init__( "Please use List[Field] instead for the schema", DeprecationWarning, ) - self.schema = schema + schemaList = [] + for key, valueType in schema.items(): + schemaList.append( + Field(name=key, dtype=VALUE_TYPES_TO_FEAST_TYPES[valueType]) + ) + self.schema = schemaList + elif isinstance(schema, List): + self.schema = schema + else: + raise Exception( + "Schema type must be either dictionary or list, not " + + str(type(schema)) + ) def validate(self, config: RepoConfig): pass @@ -499,35 +511,13 @@ def __eq__(self, other): or self.tags != other.tags ): return False - else: - if isinstance(self.schema, List) and isinstance(other.schema, List): - for field1, field2 in zip(self.schema, other.schema): - if field1 != field2: - return False - return True - elif isinstance(self.schema, Dict) and isinstance(other.schema, Dict): - for key, value in self.schema.items(): - if key not in other.schema: - return False - elif value != other.schema[key]: - return False - return True - elif isinstance(self.schema, Dict) and isinstance(other.schema, List): - dict_schema = self.schema - list_schema = other.schema - elif isinstance(self.schema, List) and isinstance(other.schema, Dict): - dict_schema = other.schema - list_schema = self.schema - - temp_schema = {} - for field in list_schema: - temp_schema[field.name] = field.dtype - for name, value in dict_schema.items(): - if name not in temp_schema: - return False - elif VALUE_TYPES_TO_FEAST_TYPES[value.value] != temp_schema[name]: + if isinstance(self.schema, List) and isinstance(other.schema, List): + for field1, field2 in zip(self.schema, other.schema): + if field1 != field2: return False return True + else: + return False @staticmethod def from_proto(data_source: DataSourceProto): diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 2c553401c0c..95f320745fe 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -16,7 +16,6 @@ from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FieldProto from feast.protos.feast.types.Value_pb2 import ValueType from feast.types import FeastType, from_value_type -from feast.value_type import ValueType class Field: diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index feacab17715..dd8e1ca5220 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -359,6 +359,11 @@ def get_request_data_schema(self) -> Dict[str, ValueType]: schema.update(new_schema) elif isinstance(request_source.schema, Dict): schema.update(request_source.schema) + else: + raise Exception( + "Request source schema is not correct type: " + + str(type(request_source.schema)) + ) return schema def get_transformed_features_df( diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index e6900e2d9ef..fcd9052eb17 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -20,6 +20,7 @@ from feast.types import FeastType, PrimitiveFeastType from tests.integration.feature_repos.universal.entities import location from feast.field import Field +from feast.data_source import DataSource, RequestSource def driver_feature_view( data_source: DataSource, From d404a4a937136afe46173da15537a03303a3a622 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 16:29:14 -0700 Subject: [PATCH 06/19] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/feast/request_feature_view.py | 6 ++---- .../integration/feature_repos/universal/feature_views.py | 9 ++++----- .../tests/integration/registration/test_inference.py | 2 +- .../tests/integration/registration/test_registry.py | 3 +-- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/sdk/python/feast/request_feature_view.py b/sdk/python/feast/request_feature_view.py index c7ffbbe2cea..dd80318d48d 100644 --- a/sdk/python/feast/request_feature_view.py +++ b/sdk/python/feast/request_feature_view.py @@ -5,6 +5,7 @@ from feast.base_feature_view import BaseFeatureView from feast.data_source import RequestSource from feast.feature_view_projection import FeatureViewProjection +from requests import request from feast.field import Field, from_value_type from feast.protos.feast.core.RequestFeatureView_pb2 import ( RequestFeatureView as RequestFeatureViewProto, @@ -69,10 +70,7 @@ def __init__( for name, dtype in request_data_source.schema.items() ] else: - new_features = [ - Field(name=field.name, dtype=field.dtype.to_value_type()) - for field in request_data_source.schema - ] + new_features = request_data_source.schema super().__init__( name=name, diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index fcd9052eb17..6cdb2815994 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,10 +17,10 @@ ValueType, ) from feast.data_source import DataSource, RequestSource +from feast.field import Field from feast.types import FeastType, PrimitiveFeastType from tests.integration.feature_repos.universal.entities import location -from feast.field import Field -from feast.data_source import DataSource, RequestSource + def driver_feature_view( data_source: DataSource, @@ -126,9 +126,8 @@ def similarity_feature_view( def create_conv_rate_request_source(): return RequestSource( name="conv_rate_input", - schema=[ - Field(name="val_to_add", dtype=PrimitiveFeastType.INT32), - ]) + schema=[Field(name="val_to_add", dtype=PrimitiveFeastType.INT32),], + ) def create_similarity_request_source(): diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index bd9ce1695b1..484aed783cc 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -29,7 +29,7 @@ SparkSource, ) from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import String, UnixTimestamp, PrimitiveFeastType +from feast.types import PrimitiveFeastType, String, UnixTimestamp from tests.utils.data_source_utils import ( prep_file_source, simple_bq_source_using_query_arg, diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index aa90deed81d..5ef3c21f73c 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -29,9 +29,8 @@ from feast.protos.feast.types import Value_pb2 as ValueProto from feast.registry import Registry from feast.repo_config import RegistryConfig -from feast.types import Array, Bytes, Float32, Int32, Int64, String, PrimitiveFeastType +from feast.types import Array, Bytes, Float32, Int32, Int64, PrimitiveFeastType, String from feast.value_type import ValueType -from feast.field import Field @pytest.fixture From f7005d753fc23bfbdb575fae4eb55503cb6877d9 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 16:31:32 -0700 Subject: [PATCH 07/19] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/feast/request_feature_view.py | 3 +-- .../tests/integration/feature_repos/universal/feature_views.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/request_feature_view.py b/sdk/python/feast/request_feature_view.py index dd80318d48d..7248ffe9890 100644 --- a/sdk/python/feast/request_feature_view.py +++ b/sdk/python/feast/request_feature_view.py @@ -5,8 +5,7 @@ from feast.base_feature_view import BaseFeatureView from feast.data_source import RequestSource from feast.feature_view_projection import FeatureViewProjection -from requests import request -from feast.field import Field, from_value_type +from feast.field import Field from feast.protos.feast.core.RequestFeatureView_pb2 import ( RequestFeatureView as RequestFeatureViewProto, ) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 6cdb2815994..32243f8a6ac 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,7 +17,6 @@ ValueType, ) from feast.data_source import DataSource, RequestSource -from feast.field import Field from feast.types import FeastType, PrimitiveFeastType from tests.integration.feature_repos.universal.entities import location @@ -126,7 +125,7 @@ def similarity_feature_view( def create_conv_rate_request_source(): return RequestSource( name="conv_rate_input", - schema=[Field(name="val_to_add", dtype=PrimitiveFeastType.INT32),], + schema=[Field(name="val_to_add", dtype=PrimitiveFeastType.INT32)], ) From a7a770324523ef3124a5bf77ed42ad248d51642d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 16:36:42 -0700 Subject: [PATCH 08/19] Fix go Signed-off-by: Kevin Zhang --- go/internal/feast/model/ondemandfeatureview.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/internal/feast/model/ondemandfeatureview.go b/go/internal/feast/model/ondemandfeatureview.go index b8d28c6d7e7..8f35ad89c2a 100644 --- a/go/internal/feast/model/ondemandfeatureview.go +++ b/go/internal/feast/model/ondemandfeatureview.go @@ -55,8 +55,8 @@ func (fs *OnDemandFeatureView) ProjectWithFeatures(featureNames []string) (*OnDe func (fs *OnDemandFeatureView) GetRequestDataSchema() map[string]types.ValueType_Enum { schema := make(map[string]types.ValueType_Enum) for _, requestDataSource := range fs.SourceRequestDataSources { - for fieldName, fieldValueType := range requestDataSource.Schema { - schema[fieldName] = fieldValueType + for _, spec := range requestDataSource.Schema { + schema[spec.Name] = spec.ValueType } } return schema From 156b171a826d7cc85c60d4c089c492d508c1aa1f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 17:09:15 -0700 Subject: [PATCH 09/19] Fix tests Signed-off-by: Kevin Zhang --- protos/feast/types/Field.proto | 2 +- sdk/python/feast/field.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/protos/feast/types/Field.proto b/protos/feast/types/Field.proto index 57c53b3b6c5..8349263cc60 100644 --- a/protos/feast/types/Field.proto +++ b/protos/feast/types/Field.proto @@ -26,5 +26,5 @@ option go_package = "github.com/feast-dev/feast/go/protos/feast/types"; message Field { string name = 1; - feast.types.Value value = 2; + feast.types.ValueType.Enum value = 2; } diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 95f320745fe..a0caedfe857 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -14,7 +14,7 @@ from feast.feature import Feature from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FieldProto -from feast.protos.feast.types.Value_pb2 import ValueType +from feast.value_type import ValueType from feast.types import FeastType, from_value_type @@ -76,7 +76,7 @@ def from_proto(cls, field_proto: FieldProto): field_proto: FieldProto protobuf object """ value_type = ValueType(field_proto.value_type) - return cls(name=field_proto.name, dtype=from_value_type(value_type)) + return cls(name=field_proto.name, dtype=from_value_type(value_type=value_type)) @classmethod def from_feature(cls, feature: Feature): From 512e00fe43a4a7c4b5a8e29e8a1b44faac908533 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 17:10:23 -0700 Subject: [PATCH 10/19] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/field.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index a0caedfe857..f6c88f1850d 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -14,8 +14,8 @@ from feast.feature import Feature from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FieldProto -from feast.value_type import ValueType from feast.types import FeastType, from_value_type +from feast.value_type import ValueType class Field: @@ -62,8 +62,6 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" - if isinstance(self.dtype, ValueType): - return self.dtype.Enum value_type = self.dtype.to_value_type() return FieldProto(name=self.name, value_type=value_type.value) From 8d62f0c88cfa6b1e6186ff96033717f96b84a4e5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 17:28:48 -0700 Subject: [PATCH 11/19] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/on_demand_feature_view.py | 6 +++--- sdk/python/feast/types.py | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index dd8e1ca5220..7afdfaa74c5 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -420,9 +420,9 @@ def infer_features(self): ) df[f"{feature.name}"] = pd.Series(dtype=dtype) for request_data in self.source_request_sources.values(): - for feature_name, feature_type in request_data.schema.items(): - dtype = feast_value_type_to_pandas_type(feature_type) - df[f"{feature_name}"] = pd.Series(dtype=dtype) + for field in request_data.schema: + dtype = feast_value_type_to_pandas_type(field.dtype.to_value_type()) + df[f"{field.name}"] = pd.Series(dtype=dtype) output_df: pd.DataFrame = self.udf.__call__(df) inferred_features = [] for f, dt in zip(output_df.columns, output_df.dtypes): diff --git a/sdk/python/feast/types.py b/sdk/python/feast/types.py index 9ce5840b347..40c1d62e7d2 100644 --- a/sdk/python/feast/types.py +++ b/sdk/python/feast/types.py @@ -81,7 +81,10 @@ def __str__(self): return PRIMITIVE_FEAST_TYPES_TO_STRING[self.name] def __eq__(self, other): - return self.value == other.value + if isinstance(other, PrimitiveFeastType): + return self.value == other.value + else: + return False def __hash__(self): return hash((PRIMITIVE_FEAST_TYPES_TO_STRING[self.name])) From eb6bf3c2297f0bf090995205af676234a0a86cad Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 22:55:08 -0700 Subject: [PATCH 12/19] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/data_source.py | 3 + sdk/python/feast/feature_store.py | 1 + .../feature_repos/repo_configuration.py | 64 +++++++++---------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 5cb941106ae..fb0a1701a36 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -519,6 +519,9 @@ def __eq__(self, other): else: return False + def __hash__(self): + return super().__hash__() + @staticmethod def from_proto(data_source: DataSourceProto): diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 63db1514129..4ba4aade99c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -679,6 +679,7 @@ def apply( for odfv in odfvs_to_update: for v in odfv.source_request_sources.values(): + print(type(v)) data_sources_set_to_update.add(v) data_sources_to_update = list(data_sources_set_to_update) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index a99ba959537..5d7c23c795c 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -86,38 +86,38 @@ [ IntegrationTestRepoConfig(online_store=REDIS_CONFIG), # GCP configurations - IntegrationTestRepoConfig( - provider="gcp", - offline_store_creator=BigQueryDataSourceCreator, - online_store="datastore", - ), - IntegrationTestRepoConfig( - provider="gcp", - offline_store_creator=BigQueryDataSourceCreator, - online_store=REDIS_CONFIG, - ), - # AWS configurations - IntegrationTestRepoConfig( - provider="aws", - offline_store_creator=RedshiftDataSourceCreator, - online_store=DYNAMO_CONFIG, - python_feature_server=True, - ), - IntegrationTestRepoConfig( - provider="aws", - offline_store_creator=RedshiftDataSourceCreator, - online_store=REDIS_CONFIG, - ), - # Snowflake configurations - IntegrationTestRepoConfig( - provider="aws", # no list features, no feature server - offline_store_creator=SnowflakeDataSourceCreator, - online_store=REDIS_CONFIG, - ), - # Go implementation for online retrieval - IntegrationTestRepoConfig( - online_store=REDIS_CONFIG, go_feature_server=True, - ), + # IntegrationTestRepoConfig( + # provider="gcp", + # offline_store_creator=BigQueryDataSourceCreator, + # online_store="datastore", + # ), + # IntegrationTestRepoConfig( + # provider="gcp", + # offline_store_creator=BigQueryDataSourceCreator, + # online_store=REDIS_CONFIG, + # ), + # # AWS configurations + # IntegrationTestRepoConfig( + # provider="aws", + # offline_store_creator=RedshiftDataSourceCreator, + # online_store=DYNAMO_CONFIG, + # python_feature_server=True, + # ), + # IntegrationTestRepoConfig( + # provider="aws", + # offline_store_creator=RedshiftDataSourceCreator, + # online_store=REDIS_CONFIG, + # ), + # # Snowflake configurations + # IntegrationTestRepoConfig( + # provider="aws", # no list features, no feature server + # offline_store_creator=SnowflakeDataSourceCreator, + # online_store=REDIS_CONFIG, + # ), + # # Go implementation for online retrieval + # IntegrationTestRepoConfig( + # online_store=REDIS_CONFIG, go_feature_server=True, + # ), ] ) full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) From fba1388b2662854429c98dc9e0cb711d5a61c36c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 22:55:47 -0700 Subject: [PATCH 13/19] Revert Signed-off-by: Kevin Zhang --- .../feature_repos/repo_configuration.py | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 5d7c23c795c..a99ba959537 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -86,38 +86,38 @@ [ IntegrationTestRepoConfig(online_store=REDIS_CONFIG), # GCP configurations - # IntegrationTestRepoConfig( - # provider="gcp", - # offline_store_creator=BigQueryDataSourceCreator, - # online_store="datastore", - # ), - # IntegrationTestRepoConfig( - # provider="gcp", - # offline_store_creator=BigQueryDataSourceCreator, - # online_store=REDIS_CONFIG, - # ), - # # AWS configurations - # IntegrationTestRepoConfig( - # provider="aws", - # offline_store_creator=RedshiftDataSourceCreator, - # online_store=DYNAMO_CONFIG, - # python_feature_server=True, - # ), - # IntegrationTestRepoConfig( - # provider="aws", - # offline_store_creator=RedshiftDataSourceCreator, - # online_store=REDIS_CONFIG, - # ), - # # Snowflake configurations - # IntegrationTestRepoConfig( - # provider="aws", # no list features, no feature server - # offline_store_creator=SnowflakeDataSourceCreator, - # online_store=REDIS_CONFIG, - # ), - # # Go implementation for online retrieval - # IntegrationTestRepoConfig( - # online_store=REDIS_CONFIG, go_feature_server=True, - # ), + IntegrationTestRepoConfig( + provider="gcp", + offline_store_creator=BigQueryDataSourceCreator, + online_store="datastore", + ), + IntegrationTestRepoConfig( + provider="gcp", + offline_store_creator=BigQueryDataSourceCreator, + online_store=REDIS_CONFIG, + ), + # AWS configurations + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=RedshiftDataSourceCreator, + online_store=DYNAMO_CONFIG, + python_feature_server=True, + ), + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=RedshiftDataSourceCreator, + online_store=REDIS_CONFIG, + ), + # Snowflake configurations + IntegrationTestRepoConfig( + provider="aws", # no list features, no feature server + offline_store_creator=SnowflakeDataSourceCreator, + online_store=REDIS_CONFIG, + ), + # Go implementation for online retrieval + IntegrationTestRepoConfig( + online_store=REDIS_CONFIG, go_feature_server=True, + ), ] ) full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) From a25a3b5f5d913614db7c3db24f85e27bd5baea1f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 23:37:00 -0700 Subject: [PATCH 14/19] fixes Signed-off-by: Kevin Zhang --- examples/java-demo/feature_repo/driver_repo.py | 7 +++---- java/datatypes/pom.xml | 4 ++-- .../test/resources/docker-compose/feast10/definitions.py | 8 +++----- .../integration/feature_repos/universal/feature_views.py | 4 ++-- ui/feature_repo/features.py | 3 +-- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/examples/java-demo/feature_repo/driver_repo.py b/examples/java-demo/feature_repo/driver_repo.py index 78280eb08d5..ce9469647f6 100644 --- a/examples/java-demo/feature_repo/driver_repo.py +++ b/examples/java-demo/feature_repo/driver_repo.py @@ -6,7 +6,6 @@ from feast.types import Float32, Float64, Int64, String from google.protobuf.duration_pb2 import Duration from feast.field import Field -from feast.types import PrimitiveFeastType from feast import Entity, Feature, FeatureView, FileSource, ValueType @@ -36,8 +35,8 @@ input_request = RequestSource( name="vals_to_add", schema=[ - Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), - Field(name="val_to_add_2", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add", dtype=Int64), + Field(name="val_to_add_2", dtype=Int64), ], ) @@ -66,7 +65,7 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: request_data_source=RequestSource( name="driver_age", schema=[ - Field(name="driver_age", dtype=PrimitiveFeastType.INT64), + Field(name="driver_age", dtype=Int64), ], ), ) diff --git a/java/datatypes/pom.xml b/java/datatypes/pom.xml index a5c82d4c45c..05244fd2b5c 100644 --- a/java/datatypes/pom.xml +++ b/java/datatypes/pom.xml @@ -58,11 +58,11 @@ true - com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} + com.google.protobuf:protoc:3.14.0:exe:osx-x86_64 grpc-java - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:1.0.0:exe:osx-x86_64 diff --git a/java/serving/src/test/resources/docker-compose/feast10/definitions.py b/java/serving/src/test/resources/docker-compose/feast10/definitions.py index 8cfea19e0db..1152dda6de1 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/definitions.py +++ b/java/serving/src/test/resources/docker-compose/feast10/definitions.py @@ -5,13 +5,11 @@ from feast.feature_view import FeatureView from feast.field import Field from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Float64, Int64, PrimitiveFeastType +from feast.types import Float32, Float64, Int64 from feast.value_type import ValueType from google.protobuf.duration_pb2 import Duration from feast import FileSource -from feast import FileSource - file_path = "driver_stats.parquet" driver_hourly_stats = FileSource( path=file_path, @@ -44,8 +42,8 @@ input_request = RequestSource( name="vals_to_add", schema=[ - Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), - Field(name="val_to_add_2", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add", dtype=Int64), + Field(name="val_to_add_2", dtype=Int64), ], ) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 32243f8a6ac..3bc7e9de00d 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,7 +17,7 @@ ValueType, ) from feast.data_source import DataSource, RequestSource -from feast.types import FeastType, PrimitiveFeastType +from feast.types import FeastType from tests.integration.feature_repos.universal.entities import location @@ -125,7 +125,7 @@ def similarity_feature_view( def create_conv_rate_request_source(): return RequestSource( name="conv_rate_input", - schema=[Field(name="val_to_add", dtype=PrimitiveFeastType.INT32)], + schema=[Field(name="val_to_add", dtype=Int32)], ) diff --git a/ui/feature_repo/features.py b/ui/feature_repo/features.py index 0cdf224b8ff..0f74b424e7b 100644 --- a/ui/feature_repo/features.py +++ b/ui/feature_repo/features.py @@ -15,7 +15,6 @@ from feast.request_feature_view import RequestFeatureView from feast.on_demand_feature_view import on_demand_feature_view from feast.field import Field -from feast.types import PrimitiveFeastType import pandas as pd zipcode = Entity( @@ -134,7 +133,7 @@ input_request = RequestSource( name="transaction", schema=[ - Field(name="transaction_amt", dtype=PrimitiveFeastType.INT64), + Field(name="transaction_amt", dtype=Int64), ], ) From 7d073d58602ff63e8360a69a515a634b0ff04648 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 23:37:48 -0700 Subject: [PATCH 15/19] Fix lint Signed-off-by: Kevin Zhang --- .../tests/integration/feature_repos/universal/feature_views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 3bc7e9de00d..59922226446 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -124,8 +124,7 @@ def similarity_feature_view( def create_conv_rate_request_source(): return RequestSource( - name="conv_rate_input", - schema=[Field(name="val_to_add", dtype=Int32)], + name="conv_rate_input", schema=[Field(name="val_to_add", dtype=Int32)], ) From 6fcdaffdfd4a933773bbf6b3f8e3f6843abdb9ed Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 13 Apr 2022 10:27:56 -0700 Subject: [PATCH 16/19] Fix and clean up Signed-off-by: Kevin Zhang --- java/datatypes/pom.xml | 4 ++-- sdk/python/feast/feature_store.py | 1 - sdk/python/feast/on_demand_feature_view.py | 5 +---- .../tests/integration/registration/test_inference.py | 12 +++++++++--- .../tests/integration/registration/test_registry.py | 11 +++++++++-- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/java/datatypes/pom.xml b/java/datatypes/pom.xml index 05244fd2b5c..a5c82d4c45c 100644 --- a/java/datatypes/pom.xml +++ b/java/datatypes/pom.xml @@ -58,11 +58,11 @@ true - com.google.protobuf:protoc:3.14.0:exe:osx-x86_64 + com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:1.0.0:exe:osx-x86_64 + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 4ba4aade99c..63db1514129 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -679,7 +679,6 @@ def apply( for odfv in odfvs_to_update: for v in odfv.source_request_sources.values(): - print(type(v)) data_sources_set_to_update.add(v) data_sources_to_update = list(data_sources_set_to_update) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 7afdfaa74c5..230a0bbaddb 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -360,10 +360,7 @@ def get_request_data_schema(self) -> Dict[str, ValueType]: elif isinstance(request_source.schema, Dict): schema.update(request_source.schema) else: - raise Exception( - "Request source schema is not correct type: " - + str(type(request_source.schema)) - ) + raise Exception(f"Request source schema is not correct type: ${str(type(request_source.schema))}") return schema def get_transformed_features_df( diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index 484aed783cc..695778ddef3 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -224,12 +224,18 @@ def test_view_with_missing_feature(features_df: pd.DataFrame) -> pd.DataFrame: with pytest.raises(SpecifiedFeaturesNotPresentError): test_view_with_missing_feature.infer_features() - -def test_datasource_inference(): +#TODO(kevjumba): remove this in feast 0.23 when deprecating +@pytest.mark.parametrize( + "request_source_schema", [ + [Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP)], + {"some_date": ValueType.UNIX_TIMESTAMP}, + ], +) +def test_datasource_inference(request_source_schema): # Create Feature Views date_request = RequestSource( name="date_request", - schema=[Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP)], + schema=request_source_schema, ) @on_demand_feature_view( diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 5ef3c21f73c..d69cd44a792 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -237,7 +237,14 @@ def test_apply_feature_view_success(test_registry): @pytest.mark.parametrize( "test_registry", [lazy_fixture("local_registry")], ) -def test_modify_feature_views_success(test_registry): +#TODO(kevjumba): remove this in feast 0.23 when deprecating +@pytest.mark.parametrize( + "request_source_schema", [ + [Field(name="my_input_1", dtype=PrimitiveFeastType.INT32)], + {"my_input_1": ValueType.INT32}, + ], +) +def test_modify_feature_views_success(test_registry, request_source_schema): # Create Feature Views batch_source = FileSource( file_format=ParquetFormat(), @@ -248,7 +255,7 @@ def test_modify_feature_views_success(test_registry): request_source = RequestSource( name="request_source", - schema=[Field(name="my_input_1", dtype=PrimitiveFeastType.INT32)], + schema=request_source_schema, ) fv1 = FeatureView( From f91d9bd2546e068f0c796a14b9b64a8725188429 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 13 Apr 2022 10:28:39 -0700 Subject: [PATCH 17/19] Lint Signed-off-by: Kevin Zhang --- sdk/python/feast/on_demand_feature_view.py | 4 +++- .../tests/integration/registration/test_inference.py | 11 +++++------ .../tests/integration/registration/test_registry.py | 10 ++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 230a0bbaddb..790891b0781 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -360,7 +360,9 @@ def get_request_data_schema(self) -> Dict[str, ValueType]: elif isinstance(request_source.schema, Dict): schema.update(request_source.schema) else: - raise Exception(f"Request source schema is not correct type: ${str(type(request_source.schema))}") + raise Exception( + f"Request source schema is not correct type: ${str(type(request_source.schema))}" + ) return schema def get_transformed_features_df( diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index 695778ddef3..526f422e9d6 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -224,19 +224,18 @@ def test_view_with_missing_feature(features_df: pd.DataFrame) -> pd.DataFrame: with pytest.raises(SpecifiedFeaturesNotPresentError): test_view_with_missing_feature.infer_features() -#TODO(kevjumba): remove this in feast 0.23 when deprecating + +# TODO(kevjumba): remove this in feast 0.23 when deprecating @pytest.mark.parametrize( - "request_source_schema", [ + "request_source_schema", + [ [Field(name="some_date", dtype=PrimitiveFeastType.UNIX_TIMESTAMP)], {"some_date": ValueType.UNIX_TIMESTAMP}, ], ) def test_datasource_inference(request_source_schema): # Create Feature Views - date_request = RequestSource( - name="date_request", - schema=request_source_schema, - ) + date_request = RequestSource(name="date_request", schema=request_source_schema,) @on_demand_feature_view( # Note: we deliberately use positional arguments here to test that they work correctly, diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index d69cd44a792..072be15bfee 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -237,9 +237,10 @@ def test_apply_feature_view_success(test_registry): @pytest.mark.parametrize( "test_registry", [lazy_fixture("local_registry")], ) -#TODO(kevjumba): remove this in feast 0.23 when deprecating +# TODO(kevjumba): remove this in feast 0.23 when deprecating @pytest.mark.parametrize( - "request_source_schema", [ + "request_source_schema", + [ [Field(name="my_input_1", dtype=PrimitiveFeastType.INT32)], {"my_input_1": ValueType.INT32}, ], @@ -253,10 +254,7 @@ def test_modify_feature_views_success(test_registry, request_source_schema): created_timestamp_column="timestamp", ) - request_source = RequestSource( - name="request_source", - schema=request_source_schema, - ) + request_source = RequestSource(name="request_source", schema=request_source_schema,) fv1 = FeatureView( name="my_feature_view_1", From a71eaf9e44c1c491df2c1dd8c903183839be7d89 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 13 Apr 2022 12:45:36 -0700 Subject: [PATCH 18/19] Fix Signed-off-by: Kevin Zhang --- .../logging/feature_repo/data/online_store.db | Bin 0 -> 16384 bytes go/internal/feast/model/ondemandfeatureview.go | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 go/cmd/server/logging/feature_repo/data/online_store.db diff --git a/go/cmd/server/logging/feature_repo/data/online_store.db b/go/cmd/server/logging/feature_repo/data/online_store.db new file mode 100644 index 0000000000000000000000000000000000000000..b6ccea139e5aca0c0b1985409dabf0578b8dcd14 GIT binary patch literal 16384 zcmeI#K~KUk6bJBb5HThQH?Hp}2{9pl0YP^lGEmq7PIby7E^G|9j>y3SehEIHU&5o^ z0!BHwc<@60P1>gG+uHYAFCE@>J;gK)qrg>kA*#YK#3hjs!ql~>>#ppNjorjAy(!Lz z_vOlmD3ssD>87&L9SQ^>009U<00Izz00bZa0SNq$z`9u|)rPjQ?tAkIzf9RxNyIYZ zMJOkcx8z7Zg-PVEWUO2jOP=M06t4zW%dx1_yS7G@7p*@?{a$Vuo$*|GY9(j9s;^&- zQq8uFw?MfMKIbiwH*an??6lfl>rXqOcDLVde7!Pv1185BISpF6e!|;nUh2I|6_fG; z){4QRL64D!=E>eux*c@v)?iFG*7#e(KUb*NrFm3~Z9`;#EPnR&9bLMK3bV1yOIJ4Y zA`1Gb9jC{woHo_8L4g1SAOHafKmY;|fB*y_009U<;7|ol(!_B8Kh)ogydVGp2tWV= q5P$##AOHafKmY=n0Pg=d2oQh(1Rwwb2tWV=5P$##AOL~G7x)Cr+SMcg literal 0 HcmV?d00001 diff --git a/go/internal/feast/model/ondemandfeatureview.go b/go/internal/feast/model/ondemandfeatureview.go index 8f35ad89c2a..adf92356073 100644 --- a/go/internal/feast/model/ondemandfeatureview.go +++ b/go/internal/feast/model/ondemandfeatureview.go @@ -55,8 +55,8 @@ func (fs *OnDemandFeatureView) ProjectWithFeatures(featureNames []string) (*OnDe func (fs *OnDemandFeatureView) GetRequestDataSchema() map[string]types.ValueType_Enum { schema := make(map[string]types.ValueType_Enum) for _, requestDataSource := range fs.SourceRequestDataSources { - for _, spec := range requestDataSource.Schema { - schema[spec.Name] = spec.ValueType + for _, featureSpec := range requestDataSource.Schema { + schema[featureSpec.Name] = featureSpec.ValueType } } return schema From a8880cfd125afacb710c0f661343d8e910d0c1a2 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 13 Apr 2022 13:45:43 -0700 Subject: [PATCH 19/19] Patch fix, will change definitions in separate pr Signed-off-by: Kevin Zhang --- .../src/test/resources/docker-compose/feast10/definitions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/java/serving/src/test/resources/docker-compose/feast10/definitions.py b/java/serving/src/test/resources/docker-compose/feast10/definitions.py index 1152dda6de1..8ffc77c33c5 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/definitions.py +++ b/java/serving/src/test/resources/docker-compose/feast10/definitions.py @@ -41,10 +41,7 @@ input_request = RequestSource( name="vals_to_add", - schema=[ - Field(name="val_to_add", dtype=Int64), - Field(name="val_to_add_2", dtype=Int64), - ], + schema={"val_to_add": ValueType.INT64, "val_to_add_2": ValueType.INT64}, )