From 07212b574a90d8fceb5a49835f1b8584be532fea Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Sun, 20 Mar 2022 00:24:06 -0700 Subject: [PATCH 01/11] fix: Simplify DataSource.from_proto logic Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 64 ++++++++++++++++----------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 3794b797cc5..ab8eb0fd2e2 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -134,6 +134,22 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: return kinesis_options_proto +_BATCH_DATA_SOURCE_TYPES = { + "file_options": "feast.infra.offline_stores.file_source.FileSource", + "bigquery_options": "feast.infra.offline_stores.bigquery_source.BigQuerySource", + "redshift_options": "feast.infra.offline_stores.redshift_source.RedshiftSource", + "snowflake_options": "feast.infra.offline_stores.snowflake_source.SnowflakeSource", +} + + +_NON_BATCH_DATA_SOURCE_TYPES = { + "request_data_options", + "custom_options", + "kafka_options", + "kinesis_options", +} + + class DataSource(ABC): """ DataSource that can be used to source features. @@ -210,44 +226,26 @@ def from_proto(data_source: DataSourceProto) -> Any: Raises: ValueError: The type of DataSource could not be identified. """ - if data_source.data_source_class_type: - cls = get_data_source_class_from_type(data_source.data_source_class_type) - return cls.from_proto(data_source) - - if data_source.request_data_options and data_source.request_data_options.schema: - data_source_obj = RequestDataSource.from_proto(data_source) - elif data_source.file_options.file_format and data_source.file_options.file_url: - from feast.infra.offline_stores.file_source import FileSource - - data_source_obj = FileSource.from_proto(data_source) - elif ( - data_source.bigquery_options.table_ref or data_source.bigquery_options.query + option_type = data_source.WhichOneof("options") + if not option_type or ( + option_type not in _BATCH_DATA_SOURCE_TYPES + and option_type not in _NON_BATCH_DATA_SOURCE_TYPES ): - from feast.infra.offline_stores.bigquery_source import BigQuerySource - - data_source_obj = BigQuerySource.from_proto(data_source) - elif data_source.redshift_options.table or data_source.redshift_options.query: - from feast.infra.offline_stores.redshift_source import RedshiftSource - - data_source_obj = RedshiftSource.from_proto(data_source) - - elif data_source.snowflake_options.table or data_source.snowflake_options.query: - from feast.infra.offline_stores.snowflake_source import SnowflakeSource + raise ValueError("Could not identify the source type being added.") - data_source_obj = SnowflakeSource.from_proto(data_source) + if option_type in _BATCH_DATA_SOURCE_TYPES: + cls = get_data_source_class_from_type(_BATCH_DATA_SOURCE_TYPES[option_type]) + return cls.from_proto(data_source) - elif ( - data_source.kafka_options.bootstrap_servers - and data_source.kafka_options.topic - and data_source.kafka_options.message_format - ): + if option_type == "custom_options": + cls = get_data_source_class_from_type(data_source.data_source_class_type) + data_source_obj = cls.from_proto(data_source) + elif option_type == "kafka_options": data_source_obj = KafkaSource.from_proto(data_source) - elif ( - data_source.kinesis_options.record_format - and data_source.kinesis_options.region - and data_source.kinesis_options.stream_name - ): + elif option_type == "kinesis_options": data_source_obj = KinesisSource.from_proto(data_source) + elif option_type == "request_data_options": + data_source_obj = RequestDataSource.from_proto(data_source) else: raise ValueError("Could not identify the source type being added.") From e99fabe43131426271872aeb929e7ae54e89bbee Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 21 Mar 2022 10:40:40 -0700 Subject: [PATCH 02/11] simpler Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index ab8eb0fd2e2..4c3755ac126 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -134,19 +134,14 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: return kinesis_options_proto -_BATCH_DATA_SOURCE_TYPES = { +_DATA_SOURCE_OPTIONS = { "file_options": "feast.infra.offline_stores.file_source.FileSource", "bigquery_options": "feast.infra.offline_stores.bigquery_source.BigQuerySource", "redshift_options": "feast.infra.offline_stores.redshift_source.RedshiftSource", "snowflake_options": "feast.infra.offline_stores.snowflake_source.SnowflakeSource", -} - - -_NON_BATCH_DATA_SOURCE_TYPES = { - "request_data_options", - "custom_options", - "kafka_options", - "kinesis_options", + "kafka_options": "feast.data_source.KafkaSource", + "kinesis_options": "feast.data_source.KinesisSource", + "request_data_options": "feast.data_source.RequestDataSource", } @@ -228,28 +223,16 @@ def from_proto(data_source: DataSourceProto) -> Any: """ option_type = data_source.WhichOneof("options") if not option_type or ( - option_type not in _BATCH_DATA_SOURCE_TYPES - and option_type not in _NON_BATCH_DATA_SOURCE_TYPES + option_type not in list(_DATA_SOURCE_OPTIONS.keys()) + ["custom_options"] ): raise ValueError("Could not identify the source type being added.") - if option_type in _BATCH_DATA_SOURCE_TYPES: - cls = get_data_source_class_from_type(_BATCH_DATA_SOURCE_TYPES[option_type]) - return cls.from_proto(data_source) - if option_type == "custom_options": cls = get_data_source_class_from_type(data_source.data_source_class_type) - data_source_obj = cls.from_proto(data_source) - elif option_type == "kafka_options": - data_source_obj = KafkaSource.from_proto(data_source) - elif option_type == "kinesis_options": - data_source_obj = KinesisSource.from_proto(data_source) - elif option_type == "request_data_options": - data_source_obj = RequestDataSource.from_proto(data_source) - else: - raise ValueError("Could not identify the source type being added.") + return cls.from_proto(data_source) - return data_source_obj + cls = get_data_source_class_from_type(_DATA_SOURCE_OPTIONS[option_type]) + return cls.from_proto(data_source) @abstractmethod def to_proto(self) -> DataSourceProto: From a86fd8d2e6ab352d985da4e121c042773beade75 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 21 Mar 2022 23:56:42 -0700 Subject: [PATCH 03/11] use enum instead Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 4c3755ac126..5baca6d7589 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -135,13 +135,13 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: _DATA_SOURCE_OPTIONS = { - "file_options": "feast.infra.offline_stores.file_source.FileSource", - "bigquery_options": "feast.infra.offline_stores.bigquery_source.BigQuerySource", - "redshift_options": "feast.infra.offline_stores.redshift_source.RedshiftSource", - "snowflake_options": "feast.infra.offline_stores.snowflake_source.SnowflakeSource", - "kafka_options": "feast.data_source.KafkaSource", - "kinesis_options": "feast.data_source.KinesisSource", - "request_data_options": "feast.data_source.RequestDataSource", + DataSourceProto.SourceType.BATCH_FILE: "feast.infra.offline_stores.file_source.FileSource", + DataSourceProto.SourceType.BATCH_BIGQUERY: "feast.infra.offline_stores.bigquery_source.BigQuerySource", + DataSourceProto.SourceType.BATCH_REDSHIFT: "feast.infra.offline_stores.redshift_source.RedshiftSource", + DataSourceProto.SourceType.BATCH_SNOWFLAKE: "feast.infra.offline_stores.snowflake_source.SnowflakeSource", + DataSourceProto.SourceType.STREAM_KAFKA: "feast.data_source.KafkaSource", + DataSourceProto.SourceType.STREAM_KINESIS: "feast.data_source.KinesisSource", + DataSourceProto.SourceType.REQUEST_SOURCE: "feast.data_source.RequestDataSource", } @@ -221,17 +221,19 @@ def from_proto(data_source: DataSourceProto) -> Any: Raises: ValueError: The type of DataSource could not be identified. """ - option_type = data_source.WhichOneof("options") - if not option_type or ( - option_type not in list(_DATA_SOURCE_OPTIONS.keys()) + ["custom_options"] + data_source_type = data_source.type + if not data_source_type or ( + data_source_type + not in list(_DATA_SOURCE_OPTIONS.keys()) + + [DataSourceProto.SourceType.CUSTOM_SOURCE] ): raise ValueError("Could not identify the source type being added.") - if option_type == "custom_options": + if data_source_type == DataSourceProto.SourceType.CUSTOM_SOURCE: cls = get_data_source_class_from_type(data_source.data_source_class_type) return cls.from_proto(data_source) - cls = get_data_source_class_from_type(_DATA_SOURCE_OPTIONS[option_type]) + cls = get_data_source_class_from_type(_DATA_SOURCE_OPTIONS[data_source_type]) return cls.from_proto(data_source) @abstractmethod From a6f9db27009fe12d1eaf81402c7353d5876f4526 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 12:37:01 -0700 Subject: [PATCH 04/11] rebase and add push Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 1 + sdk/python/feast/go_server.py | 2 +- sdk/python/feast/transformation_server.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 5baca6d7589..0d49ce22492 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -142,6 +142,7 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: DataSourceProto.SourceType.STREAM_KAFKA: "feast.data_source.KafkaSource", DataSourceProto.SourceType.STREAM_KINESIS: "feast.data_source.KinesisSource", DataSourceProto.SourceType.REQUEST_SOURCE: "feast.data_source.RequestDataSource", + DataSourceProto.SourceType.PUSH_SOURCE: "feast.data_source.PushSource", } diff --git a/sdk/python/feast/go_server.py b/sdk/python/feast/go_server.py index 1fcbab61f08..38a350fd6d2 100644 --- a/sdk/python/feast/go_server.py +++ b/sdk/python/feast/go_server.py @@ -25,10 +25,10 @@ from subprocess import Popen from typing import Any, Dict, List, Optional, Union -import grpc from tenacity import retry, stop_after_attempt, stop_after_delay, wait_exponential import feast +import grpc from feast.errors import FeatureNameCollisionError, InvalidFeaturesParameterType from feast.feature_service import FeatureService from feast.flags_helper import is_test diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py index 83f4af749e3..8e0efd73137 100644 --- a/sdk/python/feast/transformation_server.py +++ b/sdk/python/feast/transformation_server.py @@ -2,10 +2,10 @@ import sys from concurrent import futures -import grpc import pyarrow as pa from grpc_reflection.v1alpha import reflection +import grpc from feast.errors import OnDemandFeatureViewNotFoundException from feast.feature_store import FeatureStore from feast.protos.feast.serving.TransformationService_pb2 import ( From 658f4e069a4e568ebe84759f5589018255e35d43 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 13:21:22 -0700 Subject: [PATCH 05/11] imports Signed-off-by: Achal Shah --- sdk/python/feast/go_server.py | 3 ++- sdk/python/feast/transformation_server.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/go_server.py b/sdk/python/feast/go_server.py index 38a350fd6d2..290d8891def 100644 --- a/sdk/python/feast/go_server.py +++ b/sdk/python/feast/go_server.py @@ -25,10 +25,11 @@ from subprocess import Popen from typing import Any, Dict, List, Optional, Union +import grpc + from tenacity import retry, stop_after_attempt, stop_after_delay, wait_exponential import feast -import grpc from feast.errors import FeatureNameCollisionError, InvalidFeaturesParameterType from feast.feature_service import FeatureService from feast.flags_helper import is_test diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py index 8e0efd73137..ee0796319ef 100644 --- a/sdk/python/feast/transformation_server.py +++ b/sdk/python/feast/transformation_server.py @@ -3,9 +3,9 @@ from concurrent import futures import pyarrow as pa +import grpc from grpc_reflection.v1alpha import reflection -import grpc from feast.errors import OnDemandFeatureViewNotFoundException from feast.feature_store import FeatureStore from feast.protos.feast.serving.TransformationService_pb2 import ( From 886f97aca7646c62c0d3772cf672996d69e013f7 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 13:25:25 -0700 Subject: [PATCH 06/11] imports Signed-off-by: Achal Shah --- sdk/python/feast/go_server.py | 3 +-- sdk/python/feast/transformation_server.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/go_server.py b/sdk/python/feast/go_server.py index 290d8891def..38a350fd6d2 100644 --- a/sdk/python/feast/go_server.py +++ b/sdk/python/feast/go_server.py @@ -25,11 +25,10 @@ from subprocess import Popen from typing import Any, Dict, List, Optional, Union -import grpc - from tenacity import retry, stop_after_attempt, stop_after_delay, wait_exponential import feast +import grpc from feast.errors import FeatureNameCollisionError, InvalidFeaturesParameterType from feast.feature_service import FeatureService from feast.flags_helper import is_test diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py index ee0796319ef..8e0efd73137 100644 --- a/sdk/python/feast/transformation_server.py +++ b/sdk/python/feast/transformation_server.py @@ -3,9 +3,9 @@ from concurrent import futures import pyarrow as pa -import grpc from grpc_reflection.v1alpha import reflection +import grpc from feast.errors import OnDemandFeatureViewNotFoundException from feast.feature_store import FeatureStore from feast.protos.feast.serving.TransformationService_pb2 import ( From fe3d69dddf03dd4843f32488bae392a43aae5695 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 13:27:35 -0700 Subject: [PATCH 07/11] imports Signed-off-by: Achal Shah --- sdk/python/feast/go_server.py | 2 +- sdk/python/feast/transformation_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/go_server.py b/sdk/python/feast/go_server.py index 38a350fd6d2..1fcbab61f08 100644 --- a/sdk/python/feast/go_server.py +++ b/sdk/python/feast/go_server.py @@ -25,10 +25,10 @@ from subprocess import Popen from typing import Any, Dict, List, Optional, Union +import grpc from tenacity import retry, stop_after_attempt, stop_after_delay, wait_exponential import feast -import grpc from feast.errors import FeatureNameCollisionError, InvalidFeaturesParameterType from feast.feature_service import FeatureService from feast.flags_helper import is_test diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py index 8e0efd73137..83f4af749e3 100644 --- a/sdk/python/feast/transformation_server.py +++ b/sdk/python/feast/transformation_server.py @@ -2,10 +2,10 @@ import sys from concurrent import futures +import grpc import pyarrow as pa from grpc_reflection.v1alpha import reflection -import grpc from feast.errors import OnDemandFeatureViewNotFoundException from feast.feature_store import FeatureStore from feast.protos.feast.serving.TransformationService_pb2 import ( From 772cc5c541bd652c4bb1c560ac2a90fbde3e6214 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 13:48:56 -0700 Subject: [PATCH 08/11] CR Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 27 ++++----------------------- sdk/python/feast/registry.py | 3 +++ 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 0d49ce22492..0f8969db772 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -134,18 +134,6 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: return kinesis_options_proto -_DATA_SOURCE_OPTIONS = { - DataSourceProto.SourceType.BATCH_FILE: "feast.infra.offline_stores.file_source.FileSource", - DataSourceProto.SourceType.BATCH_BIGQUERY: "feast.infra.offline_stores.bigquery_source.BigQuerySource", - DataSourceProto.SourceType.BATCH_REDSHIFT: "feast.infra.offline_stores.redshift_source.RedshiftSource", - DataSourceProto.SourceType.BATCH_SNOWFLAKE: "feast.infra.offline_stores.snowflake_source.SnowflakeSource", - DataSourceProto.SourceType.STREAM_KAFKA: "feast.data_source.KafkaSource", - DataSourceProto.SourceType.STREAM_KINESIS: "feast.data_source.KinesisSource", - DataSourceProto.SourceType.REQUEST_SOURCE: "feast.data_source.RequestDataSource", - DataSourceProto.SourceType.PUSH_SOURCE: "feast.data_source.PushSource", -} - - class DataSource(ABC): """ DataSource that can be used to source features. @@ -222,20 +210,13 @@ def from_proto(data_source: DataSourceProto) -> Any: Raises: ValueError: The type of DataSource could not be identified. """ - data_source_type = data_source.type - if not data_source_type or ( - data_source_type - not in list(_DATA_SOURCE_OPTIONS.keys()) - + [DataSourceProto.SourceType.CUSTOM_SOURCE] - ): - raise ValueError("Could not identify the source type being added.") - if data_source_type == DataSourceProto.SourceType.CUSTOM_SOURCE: + if data_source.data_source_class_type: cls = get_data_source_class_from_type(data_source.data_source_class_type) return cls.from_proto(data_source) - - cls = get_data_source_class_from_type(_DATA_SOURCE_OPTIONS[data_source_type]) - return cls.from_proto(data_source) + raise ValueError( + f"Could not identify the type for data source: {data_source.name}." + ) @abstractmethod def to_proto(self) -> DataSourceProto: diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index cb1261d8c93..0bf73fcd24c 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -320,6 +320,9 @@ def apply_data_source( del registry.data_sources[idx] data_source_proto = data_source.to_proto() data_source_proto.project = project + data_source_proto.data_source_class_type = ( + f"{data_source.__class__.__module__}.{data_source.__class__.__name__}" + ) registry.data_sources.append(data_source_proto) if commit: self.commit() From 852c27efd9c7389839b1ba66fe28871fef224aa7 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 14:05:32 -0700 Subject: [PATCH 09/11] Revert "CR" This reverts commit 989d6c55752b54d2dce8a65271d6963e13b99d1b. Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 27 +++++++++++++++++++++++---- sdk/python/feast/registry.py | 3 --- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 0f8969db772..0d49ce22492 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -134,6 +134,18 @@ def to_proto(self) -> DataSourceProto.KinesisOptions: return kinesis_options_proto +_DATA_SOURCE_OPTIONS = { + DataSourceProto.SourceType.BATCH_FILE: "feast.infra.offline_stores.file_source.FileSource", + DataSourceProto.SourceType.BATCH_BIGQUERY: "feast.infra.offline_stores.bigquery_source.BigQuerySource", + DataSourceProto.SourceType.BATCH_REDSHIFT: "feast.infra.offline_stores.redshift_source.RedshiftSource", + DataSourceProto.SourceType.BATCH_SNOWFLAKE: "feast.infra.offline_stores.snowflake_source.SnowflakeSource", + DataSourceProto.SourceType.STREAM_KAFKA: "feast.data_source.KafkaSource", + DataSourceProto.SourceType.STREAM_KINESIS: "feast.data_source.KinesisSource", + DataSourceProto.SourceType.REQUEST_SOURCE: "feast.data_source.RequestDataSource", + DataSourceProto.SourceType.PUSH_SOURCE: "feast.data_source.PushSource", +} + + class DataSource(ABC): """ DataSource that can be used to source features. @@ -210,13 +222,20 @@ def from_proto(data_source: DataSourceProto) -> Any: Raises: ValueError: The type of DataSource could not be identified. """ + data_source_type = data_source.type + if not data_source_type or ( + data_source_type + not in list(_DATA_SOURCE_OPTIONS.keys()) + + [DataSourceProto.SourceType.CUSTOM_SOURCE] + ): + raise ValueError("Could not identify the source type being added.") - if data_source.data_source_class_type: + if data_source_type == DataSourceProto.SourceType.CUSTOM_SOURCE: cls = get_data_source_class_from_type(data_source.data_source_class_type) return cls.from_proto(data_source) - raise ValueError( - f"Could not identify the type for data source: {data_source.name}." - ) + + cls = get_data_source_class_from_type(_DATA_SOURCE_OPTIONS[data_source_type]) + return cls.from_proto(data_source) @abstractmethod def to_proto(self) -> DataSourceProto: diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index 0bf73fcd24c..cb1261d8c93 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -320,9 +320,6 @@ def apply_data_source( del registry.data_sources[idx] data_source_proto = data_source.to_proto() data_source_proto.project = project - data_source_proto.data_source_class_type = ( - f"{data_source.__class__.__module__}.{data_source.__class__.__name__}" - ) registry.data_sources.append(data_source_proto) if commit: self.commit() From 03cf881da736953864fc21ded2a38502f53c62a6 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 14:19:39 -0700 Subject: [PATCH 10/11] CR Signed-off-by: Achal Shah --- sdk/python/feast/registry.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index cb1261d8c93..0bf73fcd24c 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -320,6 +320,9 @@ def apply_data_source( del registry.data_sources[idx] data_source_proto = data_source.to_proto() data_source_proto.project = project + data_source_proto.data_source_class_type = ( + f"{data_source.__class__.__module__}.{data_source.__class__.__name__}" + ) registry.data_sources.append(data_source_proto) if commit: self.commit() From 30a3cf80e75e40582f9a08acc4e101e4773627a8 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Mar 2022 15:16:48 -0700 Subject: [PATCH 11/11] cr Signed-off-by: Achal Shah --- protos/feast/core/DataSource.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 22f1c257fb7..05048c31cf4 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -71,6 +71,8 @@ message DataSource { // This is an internal field that is represents the python class for the data source object a proto object represents. // This should be set by feast, and not by users. + // The field is used primarily by custom data sources and is mandatory for them to set. Feast may set it for + // first party sources as well. string data_source_class_type = 17; // Defines options for DataSource that sources features from a file