diff --git a/Makefile b/Makefile index 4b85c0e4483..6736e64078f 100644 --- a/Makefile +++ b/Makefile @@ -310,7 +310,7 @@ format-python: cd ${ROOT_DIR}/sdk/python; python -m black --target-version py38 feast tests lint-python: - cd ${ROOT_DIR}/sdk/python; python -m mypy + cd ${ROOT_DIR}/sdk/python; python -m mypy --exclude=/tests/ --follow-imports=skip feast cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ --check-only cd ${ROOT_DIR}/sdk/python; python -m flake8 feast/ tests/ cd ${ROOT_DIR}/sdk/python; python -m black --check feast tests diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c80ded2adf0..8affea898ef 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -99,6 +99,7 @@ * [MySQL (contrib)](reference/online-stores/mysql.md) * [Rockset (contrib)](reference/online-stores/rockset.md) * [Hazelcast (contrib)](reference/online-stores/hazelcast.md) + * [ScyllaDB (contrib)](reference/online-stores/scylladb.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index f86e6f6a1df..d90bfcf1632 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -54,4 +54,6 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [hazelcast.md](hazelcast.md) {% endcontent-ref %} - +{% content-ref url="scylladb.md" %} +[scylladb.md](scylladb.md) +{% endcontent-ref %} diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md new file mode 100644 index 00000000000..e28e810e214 --- /dev/null +++ b/docs/reference/online-stores/scylladb.md @@ -0,0 +1,94 @@ +# ScyllaDB Cloud online store + +## Description + +ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast. + +The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time. + +## Getting started + +Install Feast with Cassandra support: +```bash +pip install "feast[cassandra]" +``` + +Create a new Feast project: +```bash +feast init REPO_NAME -t cassandra +``` + +### Example (ScyllaDB) + +{% code title="feature_store.yaml" %} +```yaml +project: scylla_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - 172.17.0.2 + keyspace: feast + username: scylla + password: password +``` +{% endcode %} + +### Example (ScyllaDB Cloud) + +{% code title="feature_store.yaml" %} +```yaml +project: scylla_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + keyspace: feast + username: scylla + password: password +``` +{% endcode %} + + +The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). +For a full explanation of configuration options please look at file +`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. + +Storage specifications can be found at `docs/specs/online_store_format.md`. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Cassandra plugin. + +| | Cassandra | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Resources + +* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/) +* [ScyllaDB website](https://www.scylladb.com/) +* [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index b7ce19aad9b..3421fd5d309 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import enum import warnings from abc import ABC, abstractmethod @@ -485,12 +484,12 @@ def to_proto(self) -> DataSourceProto: return data_source_proto def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError @staticmethod def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: @@ -534,12 +533,12 @@ def __init__( self.schema = schema def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError def __eq__(self, other): if not isinstance(other, RequestSource): @@ -610,12 +609,12 @@ def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: @typechecked class KinesisSource(DataSource): def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError @staticmethod def from_proto(data_source: DataSourceProto): @@ -639,7 +638,7 @@ def from_proto(data_source: DataSourceProto): @staticmethod def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: - pass + raise NotImplementedError def get_table_query_string(self) -> str: raise NotImplementedError @@ -772,12 +771,12 @@ def __hash__(self): return super().__hash__() def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError @staticmethod def from_proto(data_source: DataSourceProto): diff --git a/sdk/python/feast/feature_service.py b/sdk/python/feast/feature_service.py index c3037a55da2..7ec923205a3 100644 --- a/sdk/python/feast/feature_service.py +++ b/sdk/python/feast/feature_service.py @@ -56,7 +56,7 @@ def __init__( *, name: str, features: List[Union[FeatureView, OnDemandFeatureView]], - tags: Dict[str, str] = None, + tags: Optional[Dict[str, str]] = None, description: str = "", owner: str = "", logging_config: Optional[LoggingConfig] = None, diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 67f9662d317..f87ae7ab132 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -101,7 +101,7 @@ def __init__( name: str, source: DataSource, schema: Optional[List[Field]] = None, - entities: List[Entity] = None, + entities: Optional[List[Entity]] = None, ttl: Optional[timedelta] = timedelta(days=0), online: bool = True, description: str = "", diff --git a/sdk/python/feast/importer.py b/sdk/python/feast/importer.py index bbd592101a6..938d29fe313 100644 --- a/sdk/python/feast/importer.py +++ b/sdk/python/feast/importer.py @@ -7,7 +7,7 @@ ) -def import_class(module_name: str, class_name: str, class_type: str = None): +def import_class(module_name: str, class_name: str, class_type: str = ""): """ Dynamically loads and returns a class from a module. diff --git a/sdk/python/feast/infra/contrib/spark_kafka_processor.py b/sdk/python/feast/infra/contrib/spark_kafka_processor.py index ea55d89988a..fc4a34f17bd 100644 --- a/sdk/python/feast/infra/contrib/spark_kafka_processor.py +++ b/sdk/python/feast/infra/contrib/spark_kafka_processor.py @@ -1,10 +1,11 @@ from types import MethodType -from typing import List, Optional +from typing import List, Optional, no_type_check import pandas as pd from pyspark.sql import DataFrame, SparkSession from pyspark.sql.avro.functions import from_avro from pyspark.sql.functions import col, from_json +from pyspark.sql.streaming import StreamingQuery from feast.data_format import AvroFormat, JsonFormat from feast.data_source import KafkaSource, PushMode @@ -63,12 +64,20 @@ def __init__( self.join_keys = [fs.get_entity(entity).join_key for entity in sfv.entities] super().__init__(fs=fs, sfv=sfv, data_source=sfv.stream_source) - def ingest_stream_feature_view(self, to: PushMode = PushMode.ONLINE) -> None: + # Type hinting for data_source type. + # data_source type has been checked to be an instance of KafkaSource. + self.data_source: KafkaSource = self.data_source # type: ignore + + def ingest_stream_feature_view( + self, to: PushMode = PushMode.ONLINE + ) -> StreamingQuery: ingested_stream_df = self._ingest_stream_data() transformed_df = self._construct_transformation_plan(ingested_stream_df) online_store_query = self._write_stream_data(transformed_df, to) return online_store_query + # In the line 64 of __init__(), the "data_source" is assigned a stream_source (and has to be KafkaSource as in line 40). + @no_type_check def _ingest_stream_data(self) -> StreamTable: """Only supports json and avro formats currently.""" if self.format == "json": @@ -122,7 +131,7 @@ def _ingest_stream_data(self) -> StreamTable: def _construct_transformation_plan(self, df: StreamTable) -> StreamTable: return self.sfv.udf.__call__(df) if self.sfv.udf else df - def _write_stream_data(self, df: StreamTable, to: PushMode): + def _write_stream_data(self, df: StreamTable, to: PushMode) -> StreamingQuery: # Validation occurs at the fs.write_to_online_store() phase against the stream feature view schema. def batch_write(row: DataFrame, batch_id: int): rows: pd.DataFrame = row.toPandas() diff --git a/sdk/python/feast/infra/contrib/stream_processor.py b/sdk/python/feast/infra/contrib/stream_processor.py index 24817c82eaa..c4620f4ca1d 100644 --- a/sdk/python/feast/infra/contrib/stream_processor.py +++ b/sdk/python/feast/infra/contrib/stream_processor.py @@ -1,8 +1,9 @@ -from abc import ABC +from abc import ABC, abstractmethod from types import MethodType from typing import TYPE_CHECKING, Optional from pyspark.sql import DataFrame +from typing_extensions import TypeAlias from feast.data_source import DataSource, PushMode from feast.importer import import_class @@ -17,7 +18,7 @@ } # TODO: support more types other than just Spark. -StreamTable = DataFrame +StreamTable: TypeAlias = DataFrame class ProcessorConfig(FeastConfigBaseModel): @@ -49,33 +50,37 @@ def __init__( self.sfv = sfv self.data_source = data_source + @abstractmethod def ingest_stream_feature_view(self, to: PushMode = PushMode.ONLINE) -> None: """ Ingests data from the stream source attached to the stream feature view; transforms the data and then persists it to the online store and/or offline store, depending on the 'to' parameter. """ - pass + raise NotImplementedError + @abstractmethod def _ingest_stream_data(self) -> StreamTable: """ Ingests data into a StreamTable. """ - pass + raise NotImplementedError + @abstractmethod def _construct_transformation_plan(self, table: StreamTable) -> StreamTable: """ Applies transformations on top of StreamTable object. Since stream engines use lazy evaluation, the StreamTable will not be materialized until it is actually evaluated. For example: df.collect() in spark or tbl.execute() in Flink. """ - pass + raise NotImplementedError + @abstractmethod def _write_stream_data(self, table: StreamTable, to: PushMode) -> None: """ Launches a job to persist stream data to the online store and/or offline store, depending on the 'to' parameter, and returns a handle for the job. """ - pass + raise NotImplementedError def get_stream_processor_object( diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/config.py b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py index 31dd879af6d..946831a18fb 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/config.py +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py @@ -1,5 +1,6 @@ +from typing import Literal + from pydantic import StrictBool, StrictStr -from pydantic.typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/base_config.py b/sdk/python/feast/infra/feature_servers/base_config.py index 756dd79b438..1a348032e17 100644 --- a/sdk/python/feast/infra/feature_servers/base_config.py +++ b/sdk/python/feast/infra/feature_servers/base_config.py @@ -30,5 +30,5 @@ class BaseFeatureServerConfig(FeastConfigBaseModel): enabled: StrictBool = False """Whether the feature server should be launched.""" - feature_logging: Optional[FeatureLoggingConfig] + feature_logging: Optional[FeatureLoggingConfig] = None """ Feature logging configuration """ diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py index 8d0c269cf5d..ddcbde7924a 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py @@ -1,5 +1,6 @@ +from typing import Literal + from pydantic import StrictBool -from pydantic.typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/local_process/config.py b/sdk/python/feast/infra/feature_servers/local_process/config.py index bb2e7bdf738..3d97912e4bd 100644 --- a/sdk/python/feast/infra/feature_servers/local_process/config.py +++ b/sdk/python/feast/infra/feature_servers/local_process/config.py @@ -1,4 +1,4 @@ -from pydantic.typing import Literal +from typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 36c42cd390c..62b23dfadef 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -7,7 +7,7 @@ import click import pandas as pd from colorama import Fore, Style -from pydantic import Field, StrictStr +from pydantic import ConfigDict, Field, StrictStr from pytz import utc from tqdm import tqdm @@ -72,9 +72,7 @@ class SnowflakeMaterializationEngineConfig(FeastConfigBaseModel): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) @dataclass diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 0ee82a908ed..68420c06642 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -10,6 +10,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -19,8 +20,7 @@ import pandas as pd import pyarrow import pyarrow.parquet -from pydantic import ConstrainedStr, StrictStr, validator -from pydantic.typing import Literal +from pydantic import StrictStr, field_validator from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed from feast import flags_helper @@ -72,13 +72,6 @@ def get_http_client_info(): return http_client_info.ClientInfo(user_agent=get_user_agent()) -class BigQueryTableCreateDisposition(ConstrainedStr): - """Custom constraint for table_create_disposition. To understand more, see: - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition""" - - values = {"CREATE_NEVER", "CREATE_IF_NEEDED"} - - class BigQueryOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for GCP BigQuery""" @@ -102,10 +95,15 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" - table_create_disposition: Optional[BigQueryTableCreateDisposition] = None - """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED.""" + table_create_disposition: Literal[ + "CREATE_NEVER", "CREATE_IF_NEEDED" + ] = "CREATE_IF_NEEDED" + """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED. + Custom constraint for table_create_disposition. To understand more, see: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition + """ - @validator("billing_project_id") + @field_validator("billing_project_id") def project_id_exists(cls, v, values, **kwargs): if v and not values["project_id"]: raise ValueError( diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py index 85a61106aaf..ae510171db9 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py @@ -8,6 +8,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -18,7 +19,6 @@ import pyarrow import pyarrow as pa from pydantic import StrictStr -from pydantic.typing import Literal from pytz import utc from feast import OnDemandFeatureView diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py index 8e9e3893f3a..0aca42cd682 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py @@ -297,9 +297,9 @@ class SavedDatasetAthenaStorage(SavedDatasetStorage): def __init__( self, table_ref: str, - query: str = None, - database: str = None, - data_source: str = None, + query: Optional[str] = None, + database: Optional[str] = None, + data_source: Optional[str] = None, ): self.athena_options = AthenaOptions( table=table_ref, query=query, database=database, data_source=data_source diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index 384ab69e81f..a8f05c1c4b6 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -43,15 +43,16 @@ def __init__(self, project_name: str, *args, **kwargs): workgroup=workgroup, s3_staging_location=f"s3://{bucket_name}/test_dir", ) + self, def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: table_name = destination_name diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py index 849d5cc797f..67bae292c37 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py @@ -3,7 +3,7 @@ import warnings from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union import numpy as np import pandas @@ -11,7 +11,6 @@ import pyarrow as pa import sqlalchemy from pydantic.types import StrictStr -from pydantic.typing import Literal from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker @@ -32,7 +31,7 @@ from feast.infra.provider import RetrievalJob from feast.infra.registry.base_registry import BaseRegistry from feast.on_demand_feature_view import OnDemandFeatureView -from feast.repo_config import FeastBaseModel, RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import pa_to_mssql_type from feast.usage import log_exceptions_and_usage @@ -43,7 +42,7 @@ EntitySchema = Dict[str, np.dtype] -class MsSqlServerOfflineStoreConfig(FeastBaseModel): +class MsSqlServerOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for SQL Server""" type: Literal["mssql"] = "mssql" diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py index 9b751d98efe..71ce56bdefd 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict, List, Optional import pandas as pd import pytest @@ -64,10 +64,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, - **kwargs, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: # Make sure the field mapping is correct and convert the datetime datasources. if timestamp_field in df: @@ -99,10 +99,10 @@ def create_data_source( ) def create_saved_dataset_destination(self) -> SavedDatasetStorage: - pass + raise NotImplementedError def get_prefixed_table_name(self, destination_name: str) -> str: return f"{self.project_name}_{destination_name}" def teardown(self): - pass + raise NotImplementedError diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index c2e95a8648e..9b300d7bf46 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -9,6 +9,7 @@ Iterator, KeysView, List, + Literal, Optional, Tuple, Union, @@ -19,7 +20,6 @@ import pyarrow as pa from jinja2 import BaseLoader, Environment from psycopg2 import sql -from pydantic.typing import Literal from pytz import utc from feast.data_source import DataSource diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py index f4479501323..46d5c20e977 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py @@ -82,10 +82,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py index 71c07b20c27..b9785218857 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py @@ -2,13 +2,14 @@ import shutil import tempfile import uuid -from typing import Dict, List +from typing import Dict, List, Optional import pandas as pd from pyspark import SparkConf from pyspark.sql import SparkSession from feast.data_source import DataSource +from feast.feature_logging import LoggingDestination from feast.infra.offline_stores.contrib.spark_offline_store.spark import ( SparkOfflineStoreConfig, ) @@ -68,10 +69,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, - **kwargs, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: if timestamp_field in df: df[timestamp_field] = pd.to_datetime(df[timestamp_field], utc=True) @@ -119,3 +120,7 @@ def create_saved_dataset_destination(self) -> SavedDatasetSparkStorage: def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}_{suffix}" + + def create_logged_features_destination(self) -> LoggingDestination: + # No implementation of LoggingDestination for Spark offline store. + return None # type: ignore diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py index 7d31aa90fb4..a31d368ea11 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py @@ -8,6 +8,6 @@ FULL_REPO_CONFIGS = [ IntegrationTestRepoConfig( provider="local", - offline_store_creator=TrinoSourceCreator, + offline_store_creator=TrinoSourceCreator, # type: ignore ), ] diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py index a5aa53df7ab..fcc0c8d0fa7 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py @@ -81,10 +81,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) self.client.execute_query( @@ -128,4 +128,6 @@ def create_offline_store_config(self) -> FeastConfigBaseModel: catalog="memory", dataset=self.project_name, connector={"type": "memory"}, + user="test", + auth=None, ) diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index d4cfdb66329..cdc94350244 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd import pyarrow -from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, root_validator +from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, model_validator from trino.auth import ( BasicAuthentication, CertificateAuthentication, @@ -98,14 +98,14 @@ class AuthConfig(FeastConfigBaseModel): type: Literal["kerberos", "basic", "jwt", "oauth2", "certificate"] config: Optional[Dict[StrictStr, Any]] - @root_validator - def config_only_nullable_for_oauth2(cls, values): - auth_type = values["type"] - auth_config = values["config"] + @model_validator(mode="after") + def config_only_nullable_for_oauth2(self): + auth_type = self.type + auth_config = self.config if auth_type != "oauth2" and auth_config is None: raise ValueError(f"config cannot be null for auth type '{auth_type}'") - return values + return self def to_trino_auth(self): auth_type = self.type diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 5e4107545f0..0e5064ba785 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -2,7 +2,7 @@ import uuid from datetime import datetime from pathlib import Path -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Literal, Optional, Tuple, Union import dask.dataframe as dd import pandas as pd @@ -10,7 +10,6 @@ import pyarrow.dataset import pyarrow.parquet import pytz -from pydantic.typing import Literal from feast.data_source import DataSource from feast.errors import ( diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index ac824b359f4..887b4100796 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -183,7 +183,7 @@ def create_filesystem_and_path( return None, path def get_table_query_string(self) -> str: - pass + raise NotImplementedError class FileOptions: diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 6141e3c435b..ea0682ab7ee 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import warnings -from abc import ABC, abstractmethod +from abc import ABC from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union @@ -146,13 +146,12 @@ def to_arrow( return pyarrow.Table.from_pandas(features_df) - def to_sql(self) -> str: + def to_sql(self) -> str: # type: ignore """ Return RetrievalJob generated SQL statement if applicable. """ pass - @abstractmethod def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: """ Synchronously executes the underlying query and returns the result as a pandas dataframe. @@ -162,9 +161,8 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: Does not handle on demand transformations or dataset validation. For either of those, `to_df` should be used. """ - pass + raise NotImplementedError - @abstractmethod def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: """ Synchronously executes the underlying query and returns the result as an arrow table. @@ -174,21 +172,18 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: Does not handle on demand transformations or dataset validation. For either of those, `to_arrow` should be used. """ - pass + raise NotImplementedError @property - @abstractmethod def full_feature_names(self) -> bool: """Returns True if full feature names should be applied to the results of the query.""" - pass + raise NotImplementedError @property - @abstractmethod def on_demand_feature_views(self) -> List[OnDemandFeatureView]: """Returns a list containing all the on demand feature views to be handled.""" - pass + raise NotImplementedError - @abstractmethod def persist( self, storage: SavedDatasetStorage, @@ -204,13 +199,12 @@ def persist( allow_overwrite: If True, a pre-existing location (e.g. table or file) can be overwritten. Currently not all individual offline store implementations make use of this parameter. """ - pass + raise NotImplementedError @property - @abstractmethod def metadata(self) -> Optional[RetrievalMetadata]: """Returns metadata about the retrieval job.""" - pass + raise NotImplementedError def supports_remote_storage_export(self) -> bool: """Returns True if the RetrievalJob supports `to_remote_storage`.""" @@ -226,7 +220,7 @@ def to_remote_storage(self) -> List[str]: Returns: A list of parquet file paths in remote storage. """ - raise NotImplementedError() + raise NotImplementedError class OfflineStore(ABC): @@ -239,7 +233,6 @@ class OfflineStore(ABC): """ @staticmethod - @abstractmethod def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -270,10 +263,9 @@ def pull_latest_from_table_or_query( Returns: A RetrievalJob that can be executed to get the entity rows. """ - pass + raise NotImplementedError @staticmethod - @abstractmethod def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -302,10 +294,9 @@ def get_historical_features( Returns: A RetrievalJob that can be executed to get the features. """ - pass + raise NotImplementedError @staticmethod - @abstractmethod def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -334,7 +325,7 @@ def pull_all_from_table_or_query( Returns: A RetrievalJob that can be executed to get the entity rows. """ - pass + raise NotImplementedError @staticmethod def write_logged_features( @@ -358,7 +349,7 @@ def write_logged_features( logging_config: A LoggingConfig object that determines where the logs will be written. registry: The registry for the current feature store. """ - raise NotImplementedError() + raise NotImplementedError @staticmethod def offline_write_batch( @@ -377,4 +368,4 @@ def offline_write_batch( progress: Function to be called once a portion of the data has been written, used to show progress. """ - raise NotImplementedError() + raise NotImplementedError diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 837cf49655d..2565a569ad1 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -9,6 +9,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -19,8 +20,7 @@ import pyarrow import pyarrow as pa from dateutil import parser -from pydantic import StrictStr, root_validator -from pydantic.typing import Literal +from pydantic import StrictStr, model_validator from pytz import utc from feast import OnDemandFeatureView, RedshiftSource @@ -51,13 +51,13 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): type: Literal["redshift"] = "redshift" """ Offline store type selector""" - cluster_id: Optional[StrictStr] + cluster_id: Optional[StrictStr] = None """ Redshift cluster identifier, for provisioned clusters """ - user: Optional[StrictStr] + user: Optional[StrictStr] = None """ Redshift user name, only required for provisioned clusters """ - workgroup: Optional[StrictStr] + workgroup: Optional[StrictStr] = None """ Redshift workgroup identifier, for serverless """ region: StrictStr @@ -72,16 +72,16 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): iam_role: StrictStr """ IAM Role for Redshift, granting it access to S3 """ - @root_validator - def require_cluster_and_user_or_workgroup(cls, values): + @model_validator(mode="after") + def require_cluster_and_user_or_workgroup(self): """ Provisioned Redshift clusters: Require cluster_id and user, ignore workgroup Serverless Redshift: Require workgroup, ignore cluster_id and user """ cluster_id, user, workgroup = ( - values.get("cluster_id"), - values.get("user"), - values.get("workgroup"), + self.cluster_id, + self.user, + self.workgroup, ) if not (cluster_id and user) and not workgroup: raise ValueError( @@ -90,7 +90,7 @@ def require_cluster_and_user_or_workgroup(cls, values): elif cluster_id and workgroup: raise ValueError("cannot specify both cluster_id and workgroup") - return values + return self class RedshiftOfflineStore(OfflineStore): diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index dd13ffc96c7..66e7e78651d 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -14,6 +14,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -23,8 +24,7 @@ import numpy as np import pandas as pd import pyarrow -from pydantic import Field, StrictStr -from pydantic.typing import Literal +from pydantic import ConfigDict, Field, StrictStr from pytz import utc from feast import OnDemandFeatureView @@ -119,9 +119,7 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel): convert_timestamp_columns: Optional[bool] = None """ Convert timestamp columns on export to a Parquet-supported format """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeOfflineStore(OfflineStore): diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index 0cbf82dd1c1..a8a7539ed32 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -1,5 +1,5 @@ import warnings -from typing import Callable, Dict, Iterable, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, no_type_check from typeguard import typechecked @@ -202,6 +202,7 @@ def get_table_query_string(self) -> str: def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.snowflake_type_to_feast_value_type + @no_type_check def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: @@ -223,7 +224,7 @@ def get_table_column_names_and_types( query = f"SELECT * FROM {self.get_table_query_string()} LIMIT 5" cursor = execute_snowflake_statement(conn, query) - metadata = [ + metadata: List[Dict[str, Any]] = [ { "column_name": column.name, "type_code": column.type_code, @@ -279,12 +280,12 @@ def get_table_column_names_and_types( else: row["snowflake_type"] = "NUMBERwSCALE" - elif row["type_code"] in [5, 9, 12]: + elif row["type_code"] in {5, 9, 12}: error = snowflake_unsupported_map[row["type_code"]] raise NotImplementedError( f"The following Snowflake Data Type is not supported: {error}" ) - elif row["type_code"] in [1, 2, 3, 4, 6, 7, 8, 10, 11, 13]: + elif row["type_code"] in {1, 2, 3, 4, 6, 7, 8, 10, 11, 13}: row["snowflake_type"] = snowflake_type_code_map[row["type_code"]] else: raise NotImplementedError( @@ -292,7 +293,8 @@ def get_table_column_names_and_types( ) return [ - (column["column_name"], column["snowflake_type"]) for column in metadata + (str(column["column_name"]), str(column["snowflake_type"])) + for column in metadata ] diff --git a/sdk/python/feast/infra/online_stores/bigtable.py b/sdk/python/feast/infra/online_stores/bigtable.py index 30561d0840f..3a83d23cedb 100644 --- a/sdk/python/feast/infra/online_stores/bigtable.py +++ b/sdk/python/feast/infra/online_stores/bigtable.py @@ -2,13 +2,12 @@ import logging from concurrent import futures from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Set, Tuple import google from google.cloud import bigtable from google.cloud.bigtable import row_filters from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, utils from feast.feature_view import DUMMY_ENTITY_NAME diff --git a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py index 34a8cab036d..c672e18db03 100644 --- a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py @@ -20,7 +20,17 @@ import logging from datetime import datetime -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Literal, + Optional, + Sequence, + Tuple, +) from cassandra.auth import PlainTextAuthProvider from cassandra.cluster import ( @@ -34,7 +44,6 @@ from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy from cassandra.query import PreparedStatement from pydantic import StrictFloat, StrictInt, StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig from feast.infra.key_encoding_utils import serialize_entity_key diff --git a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py index 1da9de89a81..4b2d8ae39c2 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py +++ b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py @@ -1,12 +1,11 @@ import calendar import struct from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple from happybase import ConnectionPool from happybase.connection import DEFAULT_PROTOCOL, DEFAULT_TRANSPORT from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index c09cb126f0c..cf07d5fef12 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import pymysql import pytz @@ -23,7 +23,7 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): NOTE: The class *must* end with the `OnlineStoreConfig` suffix. """ - type = "mysql" + type: Literal["mysql"] = "mysql" host: Optional[StrictStr] = None user: Optional[StrictStr] = None diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index a12e66f1090..308528aaec2 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -2,14 +2,13 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import psycopg2 import pytz from psycopg2 import sql from psycopg2.extras import execute_values from psycopg2.pool import SimpleConnectionPool -from pydantic.schema import Literal from feast import Entity from feast.feature_view import FeatureView @@ -99,6 +98,7 @@ def online_write_batch( cur_batch, page_size=batch_size, ) + conn.commit() if progress: progress(len(cur_batch)) diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index ed4e7612ba5..ae96e16c640 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -17,10 +17,19 @@ from multiprocessing.pool import ThreadPool from queue import Empty, Queue from threading import Lock, Thread -from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + Sequence, + Tuple, +) from pydantic import PositiveInt, StrictStr -from pydantic.typing import Literal from feast import Entity, utils from feast.errors import FeastProviderLoginError diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 525978e736b..a049189de7f 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -14,10 +14,9 @@ import itertools import logging from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union from pydantic import StrictBool, StrictStr -from pydantic.typing import Literal, Union from feast import Entity, FeatureView, utils from feast.infra.infra_object import DYNAMODB_INFRA_OBJECT_CLASS_TYPE, InfraObject @@ -288,12 +287,12 @@ def _get_dynamodb_resource(self, region: str, endpoint_url: Optional[str] = None ) return self._dynamodb_resource - def _sort_dynamodb_response(self, responses: list, order: list): + def _sort_dynamodb_response(self, responses: list, order: list) -> Any: """DynamoDB Batch Get Item doesn't return items in a particular order.""" # Assign an index to order order_with_index = {value: idx for idx, value in enumerate(order)} # Sort table responses by index - table_responses_ordered = [ + table_responses_ordered: Any = [ (order_with_index[tbl_res["entity_id"]], tbl_res) for tbl_res in responses ] table_responses_ordered = sorted( diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 9561705aaac..ad84e8db7c9 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -21,6 +21,7 @@ Callable, Dict, List, + Literal, Optional, Sequence, Tuple, @@ -30,7 +31,6 @@ import pytz from google.protobuf.timestamp_pb2 import Timestamp from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig, utils from feast.infra.online_stores.helpers import _mmh3, _redis_key, _redis_key_prefix diff --git a/sdk/python/feast/infra/online_stores/snowflake.py b/sdk/python/feast/infra/online_stores/snowflake.py index c1a03a2862c..f5600249c91 100644 --- a/sdk/python/feast/infra/online_stores/snowflake.py +++ b/sdk/python/feast/infra/online_stores/snowflake.py @@ -2,11 +2,10 @@ import os from binascii import hexlify from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import pandas as pd -from pydantic import Field, StrictStr -from pydantic.schema import Literal +from pydantic import ConfigDict, Field, StrictStr from feast.entity import Entity from feast.feature_view import FeatureView @@ -57,9 +56,7 @@ class SnowflakeOnlineStoreConfig(FeastConfigBaseModel): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeOnlineStore(OnlineStore): diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 6949b2bf247..4a6aa28889d 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -16,10 +16,9 @@ import sqlite3 from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple from pydantic import StrictStr -from pydantic.schema import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 28b10c12595..aca18f4856b 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -70,7 +70,7 @@ def batch_engine(self) -> BatchMaterializationEngine: if self._batch_engine: return self._batch_engine else: - engine_config = self.repo_config._batch_engine_config + engine_config = self.repo_config.batch_engine_config config_is_dict = False if isinstance(engine_config, str): engine_config_type = engine_config @@ -180,7 +180,7 @@ def online_read( config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str] = None, + requested_features: Optional[List[str]] = None, ) -> List: set_usage_attribute("provider", self.__class__.__name__) result = [] diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 82879b264af..2a9670cacef 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -211,7 +211,7 @@ def online_read( config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str] = None, + requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: """ Reads features values for the given entity keys. diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index 8928a5800dd..3d9d0c10915 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -51,6 +51,7 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def delete_entity(self, name: str, project: str, commit: bool = True): @@ -62,6 +63,7 @@ def delete_entity(self, name: str, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: @@ -77,6 +79,7 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti Returns either the specified entity, or raises an exception if none is found """ + pass @abstractmethod def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: @@ -90,6 +93,7 @@ def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity] Returns: List of entities """ + pass # Data source operations @abstractmethod @@ -104,6 +108,7 @@ def apply_data_source( project: Feast project that this data source belongs to commit: Whether to immediately commit to the registry """ + pass @abstractmethod def delete_data_source(self, name: str, project: str, commit: bool = True): @@ -115,6 +120,7 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): project: Feast project that this data source belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def get_data_source( @@ -131,6 +137,7 @@ def get_data_source( Returns: Returns either the specified data source, or raises an exception if none is found """ + pass @abstractmethod def list_data_sources( @@ -146,6 +153,7 @@ def list_data_sources( Returns: List of data sources """ + pass # Feature service operations @abstractmethod @@ -159,6 +167,7 @@ def apply_feature_service( feature_service: A feature service that will be registered project: Feast project that this entity belongs to """ + pass @abstractmethod def delete_feature_service(self, name: str, project: str, commit: bool = True): @@ -170,6 +179,7 @@ def delete_feature_service(self, name: str, project: str, commit: bool = True): project: Feast project that this feature service belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def get_feature_service( @@ -187,6 +197,7 @@ def get_feature_service( Returns either the specified feature service, or raises an exception if none is found """ + pass @abstractmethod def list_feature_services( @@ -202,6 +213,7 @@ def list_feature_services( Returns: List of feature services """ + pass # Feature view operations @abstractmethod @@ -216,6 +228,7 @@ def apply_feature_view( project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def delete_feature_view(self, name: str, project: str, commit: bool = True): @@ -227,6 +240,7 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ + pass # stream feature view operations @abstractmethod @@ -245,6 +259,7 @@ def get_stream_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + pass @abstractmethod def list_stream_feature_views( @@ -260,6 +275,7 @@ def list_stream_feature_views( Returns: List of stream feature views """ + pass # on demand feature view operations @abstractmethod @@ -278,6 +294,7 @@ def get_on_demand_feature_view( Returns either the specified on demand feature view, or raises an exception if none is found """ + pass @abstractmethod def list_on_demand_feature_views( @@ -293,6 +310,7 @@ def list_on_demand_feature_views( Returns: List of on demand feature views """ + pass # regular feature view operations @abstractmethod @@ -311,6 +329,7 @@ def get_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + pass @abstractmethod def list_feature_views( @@ -326,6 +345,7 @@ def list_feature_views( Returns: List of feature views """ + pass # request feature view operations @abstractmethod @@ -344,6 +364,7 @@ def get_request_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + pass @abstractmethod def list_request_feature_views( @@ -359,6 +380,7 @@ def list_request_feature_views( Returns: List of request feature views """ + pass @abstractmethod def apply_materialization( @@ -379,6 +401,7 @@ def apply_materialization( end_date (datetime): End date of the materialization interval to track commit: Whether the change should be persisted immediately """ + pass # Saved dataset operations @abstractmethod @@ -396,6 +419,7 @@ def apply_saved_dataset( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def get_saved_dataset( @@ -413,6 +437,7 @@ def get_saved_dataset( Returns either the specified SavedDataset, or raises an exception if none is found """ + pass def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = False): """ @@ -427,6 +452,7 @@ def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = Fals Returns either the specified SavedDataset, or raises an exception if none is found """ + pass @abstractmethod def list_saved_datasets( @@ -442,6 +468,7 @@ def list_saved_datasets( Returns: Returns the list of SavedDatasets """ + pass # Validation reference operations @abstractmethod @@ -459,6 +486,7 @@ def apply_validation_reference( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def delete_validation_reference(self, name: str, project: str, commit: bool = True): @@ -470,6 +498,7 @@ def delete_validation_reference(self, name: str, project: str, commit: bool = Tr project: Feast project that this object belongs to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def get_validation_reference( @@ -487,6 +516,7 @@ def get_validation_reference( Returns either the specified ValidationReference, or raises an exception if none is found """ + pass # TODO: Needs to be implemented. def list_validation_references( @@ -503,7 +533,9 @@ def list_validation_references( Returns: List of request feature views """ + raise NotImplementedError + @abstractmethod def list_project_metadata( self, project: str, allow_cache: bool = False ) -> List[ProjectMetadata]: @@ -517,6 +549,7 @@ def list_project_metadata( Returns: List of project metadata """ + raise NotImplementedError @abstractmethod def update_infra(self, infra: Infra, project: str, commit: bool = True): @@ -528,6 +561,7 @@ def update_infra(self, infra: Infra, project: str, commit: bool = True): project: Feast project that the Infra object refers to commit: Whether the change should be persisted immediately """ + pass @abstractmethod def get_infra(self, project: str, allow_cache: bool = False) -> Infra: @@ -541,6 +575,7 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra: Returns: The stored Infra object. """ + pass @abstractmethod def apply_user_metadata( @@ -565,14 +600,17 @@ def proto(self) -> RegistryProto: Returns: The registry proto object. """ + pass @abstractmethod def commit(self): """Commits the state of the registry cache to the remote registry store.""" + pass @abstractmethod def refresh(self, project: Optional[str] = None): """Refreshes the state of the registry cache by fetching the registry state from the remote registry store.""" + pass @staticmethod def _message_to_sorted_dict(message: Message) -> Dict[str, Any]: diff --git a/sdk/python/feast/infra/registry/registry_store.py b/sdk/python/feast/infra/registry/registry_store.py index c42a55cd9d2..5151fd74b27 100644 --- a/sdk/python/feast/infra/registry/registry_store.py +++ b/sdk/python/feast/infra/registry/registry_store.py @@ -17,7 +17,7 @@ def get_registry_proto(self) -> RegistryProto: Returns: Returns either the registry proto stored at the registry path, or an empty registry proto. """ - pass + raise NotImplementedError @abstractmethod def update_registry_proto(self, registry_proto: RegistryProto): @@ -40,7 +40,7 @@ def teardown(self): class NoopRegistryStore(RegistryStore): def get_registry_proto(self) -> RegistryProto: - pass + return RegistryProto() def update_registry_proto(self, registry_proto: RegistryProto): pass diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index 56c7bc1f659..d8b18692045 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -5,10 +5,9 @@ from datetime import datetime, timedelta from enum import Enum from threading import Lock -from typing import Any, Callable, List, Optional, Set, Union +from typing import Any, Callable, List, Literal, Optional, Set, Union -from pydantic import Field, StrictStr -from pydantic.schema import Literal +from pydantic import ConfigDict, Field, StrictStr import feast from feast import usage @@ -103,9 +102,7 @@ class SnowflakeRegistryConfig(RegistryConfig): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeRegistry(BaseRegistry): @@ -418,7 +415,7 @@ def _delete_object( """ cursor = execute_snowflake_statement(conn, query) - if cursor.rowcount < 1 and not_found_exception: + if cursor.rowcount and (cursor.rowcount < 1) and not_found_exception: # type: ignore raise not_found_exception(name, project) self._set_last_updated_metadata(datetime.utcnow(), project) diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile index c072ed01604..41f272c757c 100644 --- a/sdk/python/feast/infra/transformation_servers/Dockerfile +++ b/sdk/python/feast/infra/transformation_servers/Dockerfile @@ -15,7 +15,7 @@ COPY README.md README.md # Install dependencies -RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.[gcp,aws]' +RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[gcp,aws]' # Start feature transformation server CMD [ "python", "app.py" ] diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index ef83c6d1c62..c3604ee41f0 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -816,7 +816,7 @@ def execute_athena_query( database: str, workgroup: str, query: str, - temp_table: str = None, + temp_table: Optional[str] = None, ) -> str: """Execute athena statement synchronously. Waits for the query to finish. diff --git a/sdk/python/feast/infra/utils/hbase_utils.py b/sdk/python/feast/infra/utils/hbase_utils.py index d44f93f1619..72afda2ef3d 100644 --- a/sdk/python/feast/infra/utils/hbase_utils.py +++ b/sdk/python/feast/infra/utils/hbase_utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from happybase import ConnectionPool @@ -38,9 +38,9 @@ class HBaseConnector: def __init__( self, - pool: ConnectionPool = None, - host: str = None, - port: int = None, + pool: Optional[ConnectionPool] = None, + host: Optional[str] = None, + port: Optional[int] = None, connection_pool_size: int = 4, ): if pool is None: diff --git a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py index a4cda89a6f6..8548e4dbd86 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py +++ b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py @@ -43,7 +43,7 @@ class GetSnowflakeConnection: - def __init__(self, config: str, autocommit=True): + def __init__(self, config: Any, autocommit=True): self.config = config self.autocommit = autocommit diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 3461ae058bd..110b2afec2e 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -2,20 +2,19 @@ import os import warnings from pathlib import Path -from typing import Any +from typing import Any, Dict, Optional import yaml from pydantic import ( BaseModel, + ConfigDict, Field, StrictInt, StrictStr, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) -from pydantic.error_wrappers import ErrorWrapper -from pydantic.typing import Dict, Optional from feast.errors import ( FeastFeatureServerTypeInvalidError, @@ -93,17 +92,13 @@ class FeastBaseModel(BaseModel): """Feast Pydantic Configuration Class""" - class Config: - arbitrary_types_allowed = True - extra = "allow" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") class FeastConfigBaseModel(BaseModel): """Feast Pydantic Configuration Class""" - class Config: - arbitrary_types_allowed = True - extra = "forbid" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") class RegistryConfig(FeastBaseModel): @@ -112,7 +107,7 @@ class RegistryConfig(FeastBaseModel): registry_type: StrictStr = "file" """ str: Provider name or a class name that implements Registry.""" - registry_store_type: Optional[StrictStr] + registry_store_type: Optional[StrictStr] = None """ str: Provider name or a class name that implements RegistryStore. """ path: StrictStr = "" @@ -126,7 +121,7 @@ class RegistryConfig(FeastBaseModel): set to infinity by setting TTL to 0 seconds, which means the cache will only be loaded once and will never expire. Users can manually refresh the cache by calling feature_store.refresh_registry() """ - s3_additional_kwargs: Optional[Dict[str, str]] + s3_additional_kwargs: Optional[Dict[str, str]] = None """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ @@ -142,7 +137,7 @@ class RepoConfig(FeastBaseModel): provider: StrictStr """ str: local or gcp or aws """ - _registry_config: Any = Field(alias="registry", default="data/registry.db") + registry_config: Any = Field(alias="registry", default="data/registry.db") """ Configures the registry. Can be: 1. str: a path to a file based registry (a local path, or remote object storage path, e.g. a GCS URI) @@ -150,19 +145,19 @@ class RepoConfig(FeastBaseModel): 3. SnowflakeRegistryConfig: Using a Snowflake table to store the registry """ - _online_config: Any = Field(alias="online_store") + online_config: Any = Field(None, alias="online_store") """ OnlineStoreConfig: Online store configuration (optional depending on provider) """ - _offline_config: Any = Field(alias="offline_store") + offline_config: Any = Field(None, alias="offline_store") """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """ - _batch_engine_config: Any = Field(alias="batch_engine") + batch_engine_config: Any = Field(None, alias="batch_engine") """ BatchMaterializationEngine: Batch materialization configuration (optional depending on provider)""" - feature_server: Optional[Any] + feature_server: Optional[Any] = None """ FeatureServerConfig: Feature server configuration (optional depending on provider) """ - flags: Any + flags: Any = None """ Flags (deprecated field): Feature flags for experimental features """ repo_path: Optional[Path] = None @@ -187,42 +182,42 @@ def __init__(self, **data: Any): self._registry = None if "registry" not in data: raise FeastRegistryNotSetError() - self._registry_config = data["registry"] + self.registry_config = data["registry"] self._offline_store = None if "offline_store" in data: - self._offline_config = data["offline_store"] + self.offline_config = data["offline_store"] else: if data["provider"] == "local": - self._offline_config = "file" + self.offline_config = "file" elif data["provider"] == "gcp": - self._offline_config = "bigquery" + self.offline_config = "bigquery" elif data["provider"] == "aws": - self._offline_config = "redshift" + self.offline_config = "redshift" elif data["provider"] == "azure": - self._offline_config = "mssql" + self.offline_config = "mssql" self._online_store = None if "online_store" in data: - self._online_config = data["online_store"] + self.online_config = data["online_store"] else: if data["provider"] == "local": - self._online_config = "sqlite" + self.online_config = "sqlite" elif data["provider"] == "gcp": - self._online_config = "datastore" + self.online_config = "datastore" elif data["provider"] == "aws": - self._online_config = "dynamodb" + self.online_config = "dynamodb" elif data["provider"] == "rockset": - self._online_config = "rockset" + self.online_config = "rockset" self._batch_engine = None if "batch_engine" in data: - self._batch_engine_config = data["batch_engine"] + self.batch_engine_config = data["batch_engine"] elif "batch_engine_config" in data: - self._batch_engine_config = data["batch_engine_config"] + self.batch_engine_config = data["batch_engine_config"] else: # Defaults to using local in-process materialization engine. - self._batch_engine_config = "local" + self.batch_engine_config = "local" if isinstance(self.feature_server, Dict): self.feature_server = get_feature_server_config_from_type( @@ -242,71 +237,71 @@ def __init__(self, **data: Any): @property def registry(self): if not self._registry: - if isinstance(self._registry_config, Dict): - if "registry_type" in self._registry_config: + if isinstance(self.registry_config, Dict): + if "registry_type" in self.registry_config: self._registry = get_registry_config_from_type( - self._registry_config["registry_type"] - )(**self._registry_config) + self.registry_config["registry_type"] + )(**self.registry_config) else: # This may be a custom registry store, which does not need a 'registry_type' - self._registry = RegistryConfig(**self._registry_config) - elif isinstance(self._registry_config, str): + self._registry = RegistryConfig(**self.registry_config) + elif isinstance(self.registry_config, str): # User passed in just a path to file registry self._registry = get_registry_config_from_type("file")( - path=self._registry_config + path=self.registry_config ) - elif self._registry_config: - self._registry = self._registry_config + elif self.registry_config: + self._registry = self.registry_config return self._registry @property def offline_store(self): if not self._offline_store: - if isinstance(self._offline_config, Dict): + if isinstance(self.offline_config, Dict): self._offline_store = get_offline_config_from_type( - self._offline_config["type"] - )(**self._offline_config) - elif isinstance(self._offline_config, str): + self.offline_config["type"] + )(**self.offline_config) + elif isinstance(self.offline_config, str): self._offline_store = get_offline_config_from_type( - self._offline_config + self.offline_config )() - elif self._offline_config: - self._offline_store = self._offline_config + elif self.offline_config: + self._offline_store = self.offline_config return self._offline_store @property def online_store(self): if not self._online_store: - if isinstance(self._online_config, Dict): + if isinstance(self.online_config, Dict): self._online_store = get_online_config_from_type( - self._online_config["type"] - )(**self._online_config) - elif isinstance(self._online_config, str): - self._online_store = get_online_config_from_type(self._online_config)() - elif self._online_config: - self._online_store = self._online_config + self.online_config["type"] + )(**self.online_config) + elif isinstance(self.online_config, str): + self._online_store = get_online_config_from_type(self.online_config)() + elif self.online_config: + self._online_store = self.online_config return self._online_store @property def batch_engine(self): if not self._batch_engine: - if isinstance(self._batch_engine_config, Dict): + if isinstance(self.batch_engine_config, Dict): self._batch_engine = get_batch_engine_config_from_type( - self._batch_engine_config["type"] - )(**self._batch_engine_config) - elif isinstance(self._batch_engine_config, str): + self.batch_engine_config["type"] + )(**self.batch_engine_config) + elif isinstance(self.batch_engine_config, str): self._batch_engine = get_batch_engine_config_from_type( - self._batch_engine_config + self.batch_engine_config )() - elif self._batch_engine_config: + elif self.batch_engine_config: self._batch_engine = self._batch_engine return self._batch_engine - @root_validator(pre=True) + @model_validator(mode="before") @log_exceptions - def _validate_online_store_config(cls, values): + def _validate_online_store_config(cls, values: Any) -> Any: # This method will validate whether the online store configurations are set correctly. This explicit validation # is necessary because Pydantic Unions throw very verbose and cryptic exceptions. We also use this method to # impute the default online store type based on the selected provider. For the time being this method should be @@ -347,14 +342,11 @@ def _validate_online_store_config(cls, values): online_config_class = get_online_config_from_type(online_store_type) online_config_class(**values["online_store"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="online_store")], - model=RepoConfig, - ) + raise e return values - @root_validator(pre=True) - def _validate_offline_store_config(cls, values): + @model_validator(mode="before") + def _validate_offline_store_config(cls, values: Any) -> Any: # Set empty offline_store config if it isn't set explicitly if "offline_store" not in values: values["offline_store"] = dict() @@ -385,15 +377,12 @@ def _validate_offline_store_config(cls, values): offline_config_class = get_offline_config_from_type(offline_store_type) offline_config_class(**values["offline_store"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="offline_store")], - model=RepoConfig, - ) + raise e return values - @root_validator(pre=True) - def _validate_feature_server_config(cls, values): + @model_validator(mode="before") + def _validate_feature_server_config(cls, values: Any) -> Any: # Having no feature server is the default. if "feature_server" not in values: return values @@ -420,15 +409,13 @@ def _validate_feature_server_config(cls, values): ) feature_server_config_class(**values["feature_server"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="feature_server")], - model=RepoConfig, - ) + raise e return values - @validator("project") - def _validate_project_name(cls, v): + @field_validator("project") + @classmethod + def _validate_project_name(cls, v: str) -> str: from feast.repo_operations import is_valid_name if not is_valid_name(v): @@ -438,10 +425,11 @@ def _validate_project_name(cls, v): ) return v - @validator("flags") - def _validate_flags(cls, v): - if not isinstance(v, Dict): - return + @field_validator("flags") + @classmethod + def _validate_flags(cls, v: Optional[dict]) -> Optional[dict]: + if not isinstance(v, dict): + return v _logger.warning( "Flags are no longer necessary in Feast. Experimental features will log warnings instead." @@ -463,8 +451,7 @@ def write_to_path(self, repo_path: Path): sort_keys=False, ) - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class FeastConfigError(Exception): diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index e51e1e743bb..ad3e273d37b 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -51,7 +51,7 @@ import pyarrow # null timestamps get converted to -9223372036854775808 -NULL_TIMESTAMP_INT_VALUE = np.datetime64("NaT").astype(int) +NULL_TIMESTAMP_INT_VALUE: int = np.datetime64("NaT").astype(int) def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: @@ -114,7 +114,10 @@ def feast_value_type_to_pandas_type(value_type: ValueType) -> Any: def python_type_to_feast_value_type( - name: str, value: Any = None, recurse: bool = True, type_name: Optional[str] = None + name: str, + value: Optional[Any] = None, + recurse: bool = True, + type_name: Optional[str] = None, ) -> ValueType: """ Finds the equivalent Feast Value Type for a Python value. Both native @@ -321,7 +324,7 @@ def _python_datetime_to_int_timestamp( elif isinstance(value, Timestamp): int_timestamps.append(int(value.ToSeconds())) elif isinstance(value, np.datetime64): - int_timestamps.append(value.astype("datetime64[s]").astype(np.int_)) + int_timestamps.append(value.astype("datetime64[s]").astype(np.int_)) # type: ignore[attr-defined] elif isinstance(value, type(np.nan)): int_timestamps.append(NULL_TIMESTAMP_INT_VALUE) else: diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 740356907d8..5260f9998d9 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,11 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.2.0 # via # httpx # jupyter-server @@ -32,14 +34,14 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.5 +azure-core==1.30.0 # via # azure-identity # azure-storage-blob @@ -47,21 +49,21 @@ azure-identity==1.15.0 # via feast (setup.py) azure-storage-blob==12.19.0 # via feast (setup.py) -babel==2.13.1 +babel==2.14.0 # via # jupyterlab-server # sphinx -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.29.2 +boto3==1.34.39 # via # feast (setup.py) # moto -botocore==1.32.2 +botocore==1.34.39 # via # boto3 # moto @@ -74,13 +76,13 @@ build==1.0.3 # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.1 +cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.2 # via google-auth -cassandra-driver==3.28.0 +cassandra-driver==3.29.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -116,15 +118,15 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.3.2 +coverage[toml]==7.4.1 # via # coverage # pytest-cov -cryptography==41.0.6 +cryptography==41.0.7 # via # azure-identity # azure-storage-blob @@ -137,11 +139,11 @@ cryptography==41.0.6 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.11.0 +dask==2024.2.0 # via feast (setup.py) -db-dtypes==1.1.1 +db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -149,14 +151,14 @@ defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 # via testcontainers -dill==0.3.7 +dill==0.3.8 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -docker==6.1.3 +docker==7.0.0 # via # feast (setup.py) # testcontainers @@ -164,7 +166,7 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # ipython @@ -173,13 +175,13 @@ execnet==2.0.2 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 +fastavro==1.9.3 # via # feast (setup.py) # pandavro -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via @@ -201,7 +203,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.14.0 +google-api-core[grpc]==2.17.0 # via # feast (setup.py) # firebase-admin @@ -213,9 +215,9 @@ google-api-core[grpc]==2.14.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.108.0 +google-api-python-client==2.117.0 # via firebase-admin -google-auth==2.23.4 +google-auth==2.27.0 # via # google-api-core # google-api-python-client @@ -223,28 +225,28 @@ google-auth==2.23.4 # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.1 +google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 # via # feast (setup.py) # google-cloud-bigquery -google-cloud-bigquery-storage==2.22.0 +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) -google-cloud-bigtable==2.21.0 +google-cloud-bigtable==2.23.0 # via feast (setup.py) -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.18.0 +google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.13.1 +google-cloud-firestore==2.14.0 # via firebase-admin -google-cloud-storage==2.13.0 +google-cloud-storage==2.14.0 # via # feast (setup.py) # firebase-admin @@ -252,23 +254,23 @@ google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.61.0 +googleapis-common-protos[grpc]==1.62.0 # via # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.8 # via feast (setup.py) -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.12.7 +grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.59.2 +grpcio==1.60.1 # via # feast (setup.py) # google-api-core @@ -280,15 +282,15 @@ grpcio==1.59.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.60.1 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.60.1 # via feast (setup.py) -grpcio-status==1.59.2 +grpcio-status==1.60.1 # via google-api-core -grpcio-testing==1.59.2 +grpcio-testing==1.60.1 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.60.1 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -300,7 +302,7 @@ happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.3.0 # via feast (setup.py) -hiredis==2.2.3 +hiredis==2.3.2 # via feast (setup.py) httpcore==1.0.2 # via httpx @@ -310,11 +312,11 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.26.0 # via feast (setup.py) -identify==2.5.31 +identify==2.5.34 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio # httpx @@ -323,29 +325,28 @@ idna==3.4 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) - # great-expectations importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.26.0 +ipykernel==6.29.2 # via jupyterlab -ipython==8.17.2 +ipython==8.21.0 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via great-expectations isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.12.0 +isort==5.13.2 # via feast (setup.py) jedi==0.19.1 # via ipython @@ -372,7 +373,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # altair # feast (setup.py) @@ -380,14 +381,14 @@ jsonschema[format-nongpl]==4.20.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -400,24 +401,24 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.11 +jupyterlab==4.0.12 # via notebook -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -425,12 +426,12 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # nbconvert # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via great-expectations matplotlib-inline==0.1.6 # via @@ -444,25 +445,25 @@ mistune==3.0.2 # via # great-expectations # nbconvert -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.9 +moto==4.2.14 # via feast (setup.py) -msal==1.25.0 +msal==1.26.0 # via # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol -multiprocess==0.70.15 +multiprocess==0.70.16 # via bytewax -mypy==0.982 +mypy==1.8.0 # via # feast (setup.py) # sqlalchemy @@ -474,7 +475,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.16.0 # via jupyter-server nbformat==5.9.2 # via @@ -482,11 +483,11 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.6 +notebook==7.0.8 # via great-expectations notebook-shim==0.2.3 # via @@ -504,7 +505,7 @@ numpy==1.24.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server packaging==23.2 # via @@ -521,6 +522,7 @@ packaging==23.2 # jupyterlab # jupyterlab-server # marshmallow + # msal-extensions # nbconvert # pytest # snowflake-connector-python @@ -536,17 +538,17 @@ pandas==1.5.3 # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi partd==1.4.1 # via dask -pathspec==0.11.2 +pathspec==0.12.1 # via black pbr==6.0.0 # via mock -pexpect==4.8.0 +pexpect==4.9.0 # via ipython pip-tools==7.3.0 # via feast (setup.py) @@ -556,7 +558,7 @@ platformdirs==3.11.0 # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via thriftpy2 @@ -564,11 +566,11 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.18.0 +prometheus-client==0.19.0 # via jupyter-server -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython -proto-plus==1.22.3 +proto-plus==1.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -612,13 +614,13 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==14.0.1 +pyarrow==15.0.0 # via # db-dtypes # feast (setup.py) # google-cloud-bigquery # snowflake-connector-python -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -630,14 +632,16 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==2.6.1 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.2 + # via pydantic pyflakes==3.0.1 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via # feast (setup.py) # ipython @@ -647,11 +651,11 @@ pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.10 +pymssql==2.2.11 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==5.0.1 +pyodbc==5.1.0 # via feast (setup.py) pyopenssl==23.3.0 # via snowflake-connector-python @@ -663,7 +667,7 @@ pyproject-hooks==1.0.0 # via build pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.3 +pytest==7.4.4 # via # feast (setup.py) # pytest-benchmark @@ -685,7 +689,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.4.0 +pytest-xdist==3.5.0 # via feast (setup.py) python-dateutil==2.8.2 # via @@ -699,11 +703,11 @@ python-dateutil==2.8.2 # pandas # rockset # trino -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3.post1 +pytz==2024.1 # via # great-expectations # pandas @@ -718,19 +722,19 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.10.3 +regex==2023.12.25 # via feast (setup.py) requests==2.31.0 # via @@ -765,7 +769,7 @@ rfc3986-validator==0.1.1 # jupyter-events rockset==2.1.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -773,9 +777,9 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -s3transfer==0.7.0 +s3transfer==0.10.0 # via boto3 -scipy==1.11.3 +scipy==1.12.0 # via great-expectations send2trash==1.8.2 # via jupyter-server @@ -784,7 +788,6 @@ six==1.16.0 # asttokens # azure-core # bleach - # cassandra-driver # geomet # happybase # isodate @@ -800,7 +803,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.5.0 +snowflake-connector-python[pandas]==3.7.0 # via # feast (setup.py) # snowflake-connector-python @@ -809,34 +812,28 @@ sortedcontainers==2.4.0 soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via - # feast (setup.py) - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml -sphinxcontrib-applehelp==1.0.7 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.5 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -866,12 +863,12 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via snowflake-connector-python -toolz==0.12.0 +toolz==0.12.1 # via # altair # dask # partd -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -879,11 +876,11 @@ tornado==6.3.3 # jupyterlab # notebook # terminado -tqdm==4.66.1 +tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.13.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -908,28 +905,29 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.3.0.0 +types-pyopenssl==24.0.0.20240130 # via types-redis -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via # arrow # feast (setup.py) -types-pytz==2023.3.1.1 +types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.10 +types-redis==4.6.0.20240106 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==68.2.0.1 +types-setuptools==69.0.0.20240125 # via feast (setup.py) -types-tabulate==0.9.0.3 +types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # async-lru # azure-core # azure-storage-blob @@ -937,6 +935,7 @@ typing-extensions==4.8.0 # great-expectations # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # uvicorn @@ -959,8 +958,7 @@ urllib3==1.26.18 # requests # responses # rockset - # snowflake-connector-python -uvicorn[standard]==0.24.0.post1 +uvicorn[standard]==0.27.1 # via # feast (setup.py) # uvicorn @@ -974,7 +972,7 @@ volatile==2.1.0 # via bowler watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.10 +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -982,18 +980,17 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.4 +websocket-client==1.7.0 # via - # docker # jupyter-server # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 # via moto -wheel==0.41.3 +wheel==0.42.0 # via pip-tools -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets wrapt==1.16.0 # via testcontainers diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 18486d7fa9a..1f61b3fd576 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,21 +4,23 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.2.0 # via # httpx # starlette # watchfiles appdirs==1.4.4 # via fissix -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -36,35 +38,35 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.11.0 +dask==2024.2.0 # via feast (setup.py) -dill==0.3.7 +dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 +fastavro==1.9.3 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.10.0 +fsspec==2024.2.0 # via dask -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.59.2 +grpcio==1.60.1 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.60.1 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.60.1 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.60.1 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -76,14 +78,14 @@ httpcore==1.0.2 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.26.0 # via feast (setup.py) -idna==3.4 +idna==3.6 # via # anyio # httpx # requests -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) @@ -91,19 +93,19 @@ importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) -jsonschema==4.20.0 +jsonschema==4.21.1 # via feast (setup.py) -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.7.0 +mypy==1.8.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -127,7 +129,7 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.22.3 +proto-plus==1.23.0 # via feast (setup.py) protobuf==4.23.3 # via @@ -137,32 +139,34 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==14.0.1 +pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.13 +pydantic==2.6.1 # via # fastapi # feast (setup.py) -pygments==2.16.1 +pydantic-core==2.16.2 + # via pydantic +pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 # via pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -174,13 +178,13 @@ sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -190,26 +194,28 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -tqdm==4.66.1 +tqdm==4.66.2 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==4.24.0.4 +types-protobuf==4.24.0.20240129 # via mypy-protobuf -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # uvicorn -urllib3==2.1.0 +urllib3==2.2.0 # via requests -uvicorn[standard]==0.24.0.post1 +uvicorn[standard]==0.27.1 # via # feast (setup.py) # uvicorn diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 3bda9e72f9f..6d11e35e639 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -6,9 +6,11 @@ # alabaster==0.7.13 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.2.0 # via # httpx # jupyter-server @@ -32,14 +34,14 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.5 +azure-core==1.30.0 # via # azure-identity # azure-storage-blob @@ -47,7 +49,7 @@ azure-identity==1.15.0 # via feast (setup.py) azure-storage-blob==12.19.0 # via feast (setup.py) -babel==2.13.1 +babel==2.14.0 # via # jupyterlab-server # sphinx @@ -57,17 +59,17 @@ backports-zoneinfo==0.2.1 # via # trino # tzlocal -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.29.2 +boto3==1.34.39 # via # feast (setup.py) # moto -botocore==1.32.2 +botocore==1.34.39 # via # boto3 # moto @@ -80,13 +82,13 @@ build==1.0.3 # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.1 +cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.2 # via google-auth -cassandra-driver==3.28.0 +cassandra-driver==3.29.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -122,15 +124,15 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.3.2 +coverage[toml]==7.4.1 # via # coverage # pytest-cov -cryptography==41.0.6 +cryptography==41.0.7 # via # azure-identity # azure-storage-blob @@ -145,9 +147,9 @@ cryptography==41.0.6 # types-redis dask==2023.5.0 # via feast (setup.py) -db-dtypes==1.1.1 +db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -155,14 +157,14 @@ defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 # via testcontainers -dill==0.3.7 +dill==0.3.8 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -docker==6.1.3 +docker==7.0.0 # via # feast (setup.py) # testcontainers @@ -170,7 +172,7 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # pytest @@ -178,13 +180,13 @@ execnet==2.0.2 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 +fastavro==1.9.3 # via # feast (setup.py) # pandavro -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via @@ -206,7 +208,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.14.0 +google-api-core[grpc]==2.17.0 # via # feast (setup.py) # firebase-admin @@ -218,9 +220,9 @@ google-api-core[grpc]==2.14.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.108.0 +google-api-python-client==2.117.0 # via firebase-admin -google-auth==2.23.4 +google-auth==2.27.0 # via # google-api-core # google-api-python-client @@ -228,28 +230,28 @@ google-auth==2.23.4 # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.1 +google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 # via # feast (setup.py) # google-cloud-bigquery -google-cloud-bigquery-storage==2.22.0 +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) -google-cloud-bigtable==2.21.0 +google-cloud-bigtable==2.23.0 # via feast (setup.py) -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.18.0 +google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.13.1 +google-cloud-firestore==2.14.0 # via firebase-admin -google-cloud-storage==2.13.0 +google-cloud-storage==2.14.0 # via # feast (setup.py) # firebase-admin @@ -257,23 +259,23 @@ google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.61.0 +googleapis-common-protos[grpc]==1.62.0 # via # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.8 # via feast (setup.py) -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.12.7 +grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.59.2 +grpcio==1.60.1 # via # feast (setup.py) # google-api-core @@ -285,15 +287,15 @@ grpcio==1.59.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.60.1 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.60.1 # via feast (setup.py) -grpcio-status==1.59.2 +grpcio-status==1.60.1 # via google-api-core -grpcio-testing==1.59.2 +grpcio-testing==1.60.1 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.60.1 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -305,7 +307,7 @@ happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.3.0 # via feast (setup.py) -hiredis==2.2.3 +hiredis==2.3.2 # via feast (setup.py) httpcore==1.0.2 # via httpx @@ -315,11 +317,11 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.26.0 # via feast (setup.py) -identify==2.5.31 +identify==2.5.34 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio # httpx @@ -328,12 +330,11 @@ idna==3.4 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # build # dask # feast (setup.py) - # great-expectations # jupyter-client # jupyter-lsp # jupyterlab @@ -348,20 +349,20 @@ importlib-resources==6.1.1 # jupyterlab iniconfig==2.0.0 # via pytest -ipykernel==6.26.0 +ipykernel==6.29.2 # via jupyterlab ipython==8.12.3 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via great-expectations isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.12.0 +isort==5.13.2 # via feast (setup.py) jedi==0.19.1 # via ipython @@ -388,7 +389,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # altair # feast (setup.py) @@ -396,14 +397,14 @@ jsonschema[format-nongpl]==4.20.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -416,24 +417,24 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.11 +jupyterlab==4.0.12 # via notebook -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -441,12 +442,12 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # nbconvert # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via great-expectations matplotlib-inline==0.1.6 # via @@ -460,25 +461,25 @@ mistune==3.0.2 # via # great-expectations # nbconvert -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.9 +moto==4.2.14 # via feast (setup.py) -msal==1.25.0 +msal==1.26.0 # via # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol -multiprocess==0.70.15 +multiprocess==0.70.16 # via bytewax -mypy==0.982 +mypy==1.8.0 # via # feast (setup.py) # sqlalchemy @@ -490,7 +491,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.16.0 # via jupyter-server nbformat==5.9.2 # via @@ -498,11 +499,11 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.6 +notebook==7.0.8 # via great-expectations notebook-shim==0.2.3 # via @@ -520,7 +521,7 @@ numpy==1.24.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server packaging==23.2 # via @@ -537,6 +538,7 @@ packaging==23.2 # jupyterlab # jupyterlab-server # marshmallow + # msal-extensions # nbconvert # pytest # snowflake-connector-python @@ -552,17 +554,17 @@ pandas==1.5.3 # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi partd==1.4.1 # via dask -pathspec==0.11.2 +pathspec==0.12.1 # via black pbr==6.0.0 # via mock -pexpect==4.8.0 +pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython @@ -576,7 +578,7 @@ platformdirs==3.11.0 # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via thriftpy2 @@ -584,11 +586,11 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.18.0 +prometheus-client==0.19.0 # via jupyter-server -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython -proto-plus==1.22.3 +proto-plus==1.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -632,13 +634,13 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==14.0.1 +pyarrow==15.0.0 # via # db-dtypes # feast (setup.py) # google-cloud-bigquery # snowflake-connector-python -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -650,14 +652,16 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==2.6.1 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.2 + # via pydantic pyflakes==3.0.1 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via # feast (setup.py) # ipython @@ -667,11 +671,11 @@ pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.10 +pymssql==2.2.11 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==5.0.1 +pyodbc==5.1.0 # via feast (setup.py) pyopenssl==23.3.0 # via snowflake-connector-python @@ -683,7 +687,7 @@ pyproject-hooks==1.0.0 # via build pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.3 +pytest==7.4.4 # via # feast (setup.py) # pytest-benchmark @@ -705,7 +709,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.4.0 +pytest-xdist==3.5.0 # via feast (setup.py) python-dateutil==2.8.2 # via @@ -719,11 +723,11 @@ python-dateutil==2.8.2 # pandas # rockset # trino -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3.post1 +pytz==2024.1 # via # babel # great-expectations @@ -739,19 +743,19 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.10.3 +regex==2023.12.25 # via feast (setup.py) requests==2.31.0 # via @@ -786,7 +790,7 @@ rfc3986-validator==0.1.1 # jupyter-events rockset==2.1.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -796,7 +800,7 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.7.0 +s3transfer==0.10.0 # via boto3 scipy==1.10.1 # via great-expectations @@ -807,7 +811,6 @@ six==1.16.0 # asttokens # azure-core # bleach - # cassandra-driver # geomet # happybase # isodate @@ -823,7 +826,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.5.0 +snowflake-connector-python[pandas]==3.7.0 # via # feast (setup.py) # snowflake-connector-python @@ -845,15 +848,15 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -883,12 +886,12 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via snowflake-connector-python -toolz==0.12.0 +toolz==0.12.1 # via # altair # dask # partd -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -896,11 +899,11 @@ tornado==6.3.3 # jupyterlab # notebook # terminado -tqdm==4.66.1 +tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.13.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -925,28 +928,30 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.3.0.0 +types-pyopenssl==24.0.0.20240130 # via types-redis -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via # arrow # feast (setup.py) -types-pytz==2023.3.1.1 +types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.10 +types-redis==4.6.0.20240106 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==68.2.0.1 +types-setuptools==69.0.0.20240125 # via feast (setup.py) -types-tabulate==0.9.0.3 +types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # annotated-types + # anyio # async-lru # azure-core # azure-storage-blob @@ -956,6 +961,7 @@ typing-extensions==4.8.0 # ipython # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # starlette @@ -980,7 +986,7 @@ urllib3==1.26.18 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.24.0.post1 +uvicorn[standard]==0.27.1 # via # feast (setup.py) # uvicorn @@ -994,7 +1000,7 @@ volatile==2.1.0 # via bowler watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.10 +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -1002,18 +1008,17 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.4 +websocket-client==1.7.0 # via - # docker # jupyter-server # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 # via moto -wheel==0.41.3 +wheel==0.42.0 # via pip-tools -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets wrapt==1.16.0 # via testcontainers diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index c180c50c81e..6213f6fcb52 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,21 +4,23 @@ # # pip-compile --output-file=sdk/python/requirements/py3.8-requirements.txt # -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.2.0 # via # httpx # starlette # watchfiles appdirs==1.4.4 # via fissix -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -38,33 +40,33 @@ colorama==0.4.6 # via feast (setup.py) dask==2023.5.0 # via feast (setup.py) -dill==0.3.7 +dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 +fastavro==1.9.3 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.10.0 +fsspec==2024.2.0 # via dask -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.59.2 +grpcio==1.60.1 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.60.1 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.60.1 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.60.1 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -76,14 +78,14 @@ httpcore==1.0.2 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.26.0 # via feast (setup.py) -idna==3.4 +idna==3.6 # via # anyio # httpx # requests -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) @@ -94,19 +96,19 @@ importlib-resources==6.1.1 # jsonschema-specifications jinja2==3.1.3 # via feast (setup.py) -jsonschema==4.20.0 +jsonschema==4.21.1 # via feast (setup.py) -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.7.0 +mypy==1.8.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -132,7 +134,7 @@ partd==1.4.1 # via dask pkgutil-resolve-name==1.3.10 # via jsonschema -proto-plus==1.22.3 +proto-plus==1.23.0 # via feast (setup.py) protobuf==4.23.3 # via @@ -142,32 +144,34 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==14.0.1 +pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.13 +pydantic==2.6.1 # via # fastapi # feast (setup.py) -pygments==2.16.1 +pydantic-core==2.16.2 + # via pydantic +pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 # via pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -179,13 +183,13 @@ sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -195,27 +199,30 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -tqdm==4.66.1 +tqdm==4.66.2 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==4.24.0.4 +types-protobuf==4.24.0.20240129 # via mypy-protobuf -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # annotated-types + # anyio # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # starlette # uvicorn -urllib3==2.1.0 +urllib3==2.2.0 # via requests -uvicorn[standard]==0.24.0.post1 +uvicorn[standard]==0.27.1 # via # feast (setup.py) # uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 6989d5b4ccf..09f97efcd25 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,11 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.2.0 # via # httpx # jupyter-server @@ -32,14 +34,14 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.5 +azure-core==1.30.0 # via # azure-identity # azure-storage-blob @@ -47,21 +49,21 @@ azure-identity==1.15.0 # via feast (setup.py) azure-storage-blob==12.19.0 # via feast (setup.py) -babel==2.13.1 +babel==2.14.0 # via # jupyterlab-server # sphinx -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.29.2 +boto3==1.34.39 # via # feast (setup.py) # moto -botocore==1.32.2 +botocore==1.34.39 # via # boto3 # moto @@ -74,13 +76,13 @@ build==1.0.3 # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.1 +cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.2 # via google-auth -cassandra-driver==3.28.0 +cassandra-driver==3.29.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -116,15 +118,15 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.3.2 +coverage[toml]==7.4.1 # via # coverage # pytest-cov -cryptography==41.0.6 +cryptography==41.0.7 # via # azure-identity # azure-storage-blob @@ -137,11 +139,11 @@ cryptography==41.0.6 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.11.0 +dask==2024.2.0 # via feast (setup.py) -db-dtypes==1.1.1 +db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -149,14 +151,14 @@ defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 # via testcontainers -dill==0.3.7 +dill==0.3.8 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -docker==6.1.3 +docker==7.0.0 # via # feast (setup.py) # testcontainers @@ -164,7 +166,7 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # ipython @@ -173,13 +175,13 @@ execnet==2.0.2 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 +fastavro==1.9.3 # via # feast (setup.py) # pandavro -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via @@ -201,7 +203,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.14.0 +google-api-core[grpc]==2.17.0 # via # feast (setup.py) # firebase-admin @@ -213,9 +215,9 @@ google-api-core[grpc]==2.14.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.108.0 +google-api-python-client==2.117.0 # via firebase-admin -google-auth==2.23.4 +google-auth==2.27.0 # via # google-api-core # google-api-python-client @@ -223,28 +225,28 @@ google-auth==2.23.4 # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.1 +google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 # via # feast (setup.py) # google-cloud-bigquery -google-cloud-bigquery-storage==2.22.0 +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) -google-cloud-bigtable==2.21.0 +google-cloud-bigtable==2.23.0 # via feast (setup.py) -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.18.0 +google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.13.1 +google-cloud-firestore==2.14.0 # via firebase-admin -google-cloud-storage==2.13.0 +google-cloud-storage==2.14.0 # via # feast (setup.py) # firebase-admin @@ -252,23 +254,23 @@ google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.61.0 +googleapis-common-protos[grpc]==1.62.0 # via # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.8 # via feast (setup.py) -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.12.7 +grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.59.2 +grpcio==1.60.1 # via # feast (setup.py) # google-api-core @@ -280,15 +282,15 @@ grpcio==1.59.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.60.1 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.60.1 # via feast (setup.py) -grpcio-status==1.59.2 +grpcio-status==1.60.1 # via google-api-core -grpcio-testing==1.59.2 +grpcio-testing==1.60.1 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.60.1 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -300,7 +302,7 @@ happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.3.0 # via feast (setup.py) -hiredis==2.2.3 +hiredis==2.3.2 # via feast (setup.py) httpcore==1.0.2 # via httpx @@ -310,11 +312,11 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.26.0 # via feast (setup.py) -identify==2.5.31 +identify==2.5.34 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio # httpx @@ -323,12 +325,11 @@ idna==3.4 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # build # dask # feast (setup.py) - # great-expectations # jupyter-client # jupyter-lsp # jupyterlab @@ -339,20 +340,20 @@ importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.26.0 +ipykernel==6.29.2 # via jupyterlab -ipython==8.17.2 +ipython==8.18.1 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via great-expectations isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.12.0 +isort==5.13.2 # via feast (setup.py) jedi==0.19.1 # via ipython @@ -379,7 +380,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # altair # feast (setup.py) @@ -387,14 +388,14 @@ jsonschema[format-nongpl]==4.20.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -407,24 +408,24 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.11 +jupyterlab==4.0.12 # via notebook -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -432,12 +433,12 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # nbconvert # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via great-expectations matplotlib-inline==0.1.6 # via @@ -451,25 +452,25 @@ mistune==3.0.2 # via # great-expectations # nbconvert -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.9 +moto==4.2.14 # via feast (setup.py) -msal==1.25.0 +msal==1.26.0 # via # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol -multiprocess==0.70.15 +multiprocess==0.70.16 # via bytewax -mypy==0.982 +mypy==1.8.0 # via # feast (setup.py) # sqlalchemy @@ -481,7 +482,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.16.0 # via jupyter-server nbformat==5.9.2 # via @@ -489,11 +490,11 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.6 +notebook==7.0.8 # via great-expectations notebook-shim==0.2.3 # via @@ -511,7 +512,7 @@ numpy==1.24.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server packaging==23.2 # via @@ -528,6 +529,7 @@ packaging==23.2 # jupyterlab # jupyterlab-server # marshmallow + # msal-extensions # nbconvert # pytest # snowflake-connector-python @@ -543,17 +545,17 @@ pandas==1.5.3 # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi partd==1.4.1 # via dask -pathspec==0.11.2 +pathspec==0.12.1 # via black pbr==6.0.0 # via mock -pexpect==4.8.0 +pexpect==4.9.0 # via ipython pip-tools==7.3.0 # via feast (setup.py) @@ -563,7 +565,7 @@ platformdirs==3.11.0 # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via thriftpy2 @@ -571,11 +573,11 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.18.0 +prometheus-client==0.19.0 # via jupyter-server -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython -proto-plus==1.22.3 +proto-plus==1.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -619,13 +621,13 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==14.0.1 +pyarrow==15.0.0 # via # db-dtypes # feast (setup.py) # google-cloud-bigquery # snowflake-connector-python -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -637,14 +639,16 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==2.6.1 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.2 + # via pydantic pyflakes==3.0.1 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via # feast (setup.py) # ipython @@ -654,11 +658,11 @@ pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.10 +pymssql==2.2.11 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==5.0.1 +pyodbc==5.1.0 # via feast (setup.py) pyopenssl==23.3.0 # via snowflake-connector-python @@ -670,7 +674,7 @@ pyproject-hooks==1.0.0 # via build pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.3 +pytest==7.4.4 # via # feast (setup.py) # pytest-benchmark @@ -692,7 +696,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.4.0 +pytest-xdist==3.5.0 # via feast (setup.py) python-dateutil==2.8.2 # via @@ -706,11 +710,11 @@ python-dateutil==2.8.2 # pandas # rockset # trino -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3.post1 +pytz==2024.1 # via # great-expectations # pandas @@ -725,19 +729,19 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.10.3 +regex==2023.12.25 # via feast (setup.py) requests==2.31.0 # via @@ -772,7 +776,7 @@ rfc3986-validator==0.1.1 # jupyter-events rockset==2.1.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -782,9 +786,9 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.7.0 +s3transfer==0.10.0 # via boto3 -scipy==1.11.3 +scipy==1.12.0 # via great-expectations send2trash==1.8.2 # via jupyter-server @@ -793,7 +797,6 @@ six==1.16.0 # asttokens # azure-core # bleach - # cassandra-driver # geomet # happybase # isodate @@ -809,7 +812,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.5.0 +snowflake-connector-python[pandas]==3.7.0 # via # feast (setup.py) # snowflake-connector-python @@ -818,34 +821,28 @@ sortedcontainers==2.4.0 soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via - # feast (setup.py) - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml -sphinxcontrib-applehelp==1.0.7 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.5 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -875,12 +872,12 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via snowflake-connector-python -toolz==0.12.0 +toolz==0.12.1 # via # altair # dask # partd -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -888,11 +885,11 @@ tornado==6.3.3 # jupyterlab # notebook # terminado -tqdm==4.66.1 +tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.13.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -917,28 +914,29 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.3.0.0 +types-pyopenssl==24.0.0.20240130 # via types-redis -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via # arrow # feast (setup.py) -types-pytz==2023.3.1.1 +types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.10 +types-redis==4.6.0.20240106 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==68.2.0.1 +types-setuptools==69.0.0.20240125 # via feast (setup.py) -types-tabulate==0.9.0.3 +types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # async-lru # azure-core # azure-storage-blob @@ -948,6 +946,7 @@ typing-extensions==4.8.0 # ipython # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # starlette @@ -972,7 +971,7 @@ urllib3==1.26.18 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.24.0.post1 +uvicorn[standard]==0.27.1 # via # feast (setup.py) # uvicorn @@ -986,7 +985,7 @@ volatile==2.1.0 # via bowler watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.10 +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -994,18 +993,17 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.4 +websocket-client==1.7.0 # via - # docker # jupyter-server # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 # via moto -wheel==0.41.3 +wheel==0.42.0 # via pip-tools -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets wrapt==1.16.0 # via testcontainers diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 3b6f88b4e2a..3b91ebd6b2e 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,21 +4,23 @@ # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.2.0 # via # httpx # starlette # watchfiles appdirs==1.4.4 # via fissix -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -36,35 +38,35 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.11.0 +dask==2024.2.0 # via feast (setup.py) -dill==0.3.7 +dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 +fastavro==1.9.3 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.10.0 +fsspec==2024.2.0 # via dask -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.59.2 +grpcio==1.60.1 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.60.1 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.60.1 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.60.1 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -76,14 +78,14 @@ httpcore==1.0.2 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.26.0 # via feast (setup.py) -idna==3.4 +idna==3.6 # via # anyio # httpx # requests -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) @@ -91,19 +93,19 @@ importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) -jsonschema==4.20.0 +jsonschema==4.21.1 # via feast (setup.py) -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.7.0 +mypy==1.8.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -127,7 +129,7 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.22.3 +proto-plus==1.23.0 # via feast (setup.py) protobuf==4.23.3 # via @@ -137,32 +139,34 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==14.0.1 +pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.13 +pydantic==2.6.1 # via # fastapi # feast (setup.py) -pygments==2.16.1 +pydantic-core==2.16.2 + # via pydantic +pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 # via pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -174,13 +178,13 @@ sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -190,27 +194,29 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -tqdm==4.66.1 +tqdm==4.66.2 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==4.24.0.4 +types-protobuf==4.24.0.20240129 # via mypy-protobuf -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # starlette # uvicorn -urllib3==2.1.0 +urllib3==2.2.0 # via requests -uvicorn[standard]==0.24.0.post1 +uvicorn[standard]==0.27.1 # via # feast (setup.py) # uvicorn diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 728bd9b34f7..743a1ce4a0f 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -18,7 +18,7 @@ from datetime import datetime, timedelta from multiprocessing import Process from sys import platform -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, no_type_check import pandas as pd import pytest @@ -187,9 +187,10 @@ def environment(request, worker_id): e.online_store_creator.teardown() -_config_cache = {} +_config_cache: Any = {} +@no_type_check def pytest_generate_tests(metafunc: pytest.Metafunc): """ This function receives each test function (wrapped in Metafunc) diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 8d5b1979fa3..1fc66aee845 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -9,7 +9,7 @@ def create_basic_driver_dataset( entity_type: FeastType = Int32, - feature_dtype: str = None, + feature_dtype: Optional[str] = None, feature_is_list: bool = False, list_has_empty_list: bool = False, ) -> pd.DataFrame: diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index d27e2645d4e..ba256a3813c 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -71,16 +71,16 @@ def get_historical_features( project: str, full_feature_names: bool = False, ) -> RetrievalJob: - pass + return RetrievalJob() def online_read( self, config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str] = None, + requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - pass + return [] def retrieve_saved_dataset(self, config: RepoConfig, dataset: SavedDataset): pass @@ -102,4 +102,4 @@ def retrieve_feature_service_logs( config: RepoConfig, registry: BaseRegistry, ) -> RetrievalJob: - pass + return RetrievalJob() diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 027dea2c582..f745bafa132 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -99,7 +99,7 @@ "host": os.getenv("ROCKSET_APISERVER", "api.rs2.usw2.rockset.com"), } -OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, DataSourceCreator] = { +OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, Tuple[str, Type[DataSourceCreator]]] = { "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), "redshift": ("aws", RedshiftDataSourceCreator), @@ -111,7 +111,7 @@ ] AVAILABLE_ONLINE_STORES: Dict[ - str, Tuple[Union[str, Dict[str, str]], Optional[Type[OnlineStoreCreator]]] + str, Tuple[Union[str, Dict[Any, Any]], Optional[Type[OnlineStoreCreator]]] ] = { "sqlite": ({"type": "sqlite"}, None), } @@ -169,7 +169,7 @@ AVAILABLE_ONLINE_STORES = { c.online_store["type"] if isinstance(c.online_store, dict) - else c.online_store: (c.online_store, c.online_store_creator) + else c.online_store: (c.online_store, c.online_store_creator) # type: ignore for c in FULL_REPO_CONFIGS } @@ -328,7 +328,7 @@ class UniversalFeatureViews: customer: FeatureView global_fv: FeatureView driver: FeatureView - driver_odfv: OnDemandFeatureView + driver_odfv: Optional[OnDemandFeatureView] order: FeatureView location: FeatureView field_mapping: FeatureView @@ -410,9 +410,7 @@ def construct_test_environment( online_creator = test_repo_config.online_store_creator( project, fixture_request=fixture_request ) - online_store = ( - test_repo_config.online_store - ) = online_creator.create_online_store() + online_store = online_creator.create_online_store() else: online_creator = None online_store = test_repo_config.online_store @@ -422,7 +420,7 @@ def construct_test_environment( AwsLambdaFeatureServerConfig, ) - feature_server = AwsLambdaFeatureServerConfig( + feature_server: Any = AwsLambdaFeatureServerConfig( enabled=True, execution_role_name=os.getenv( "AWS_LAMBDA_ROLE", @@ -465,7 +463,7 @@ def construct_test_environment( # Create feature_store.yaml out of the config with open(Path(repo_dir_name) / "feature_store.yaml", "w") as f: - yaml.safe_dump(json.loads(config.json()), f) + yaml.safe_dump(json.loads(config.model_dump_json(by_alias=True)), f) fs = FeatureStore(repo_dir_name) # We need to initialize the registry, because if nothing is applied in the test before tearing down diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index b36af0db472..54be44fd1ff 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -20,7 +20,7 @@ def create_data_source( destination_name: str, event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = None, ) -> DataSource: """ @@ -42,19 +42,20 @@ def create_data_source( A Data source object, pointing to a table or file that is uploaded/persisted for the purpose of the test. """ - ... + pass @abstractmethod def create_offline_store_config(self) -> FeastConfigBaseModel: - ... + pass @abstractmethod def create_saved_dataset_destination(self) -> SavedDatasetStorage: - ... + pass + @abstractmethod def create_logged_features_destination(self) -> LoggingDestination: - pass + raise NotImplementedError @abstractmethod def teardown(self): - ... + pass diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 384037eef14..6122d8fd9cc 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -64,10 +64,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, - **kwargs, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 124dd4c88d6..d65740278be 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -39,9 +39,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -94,9 +95,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -167,11 +169,11 @@ def _upload_parquet_file(self, df, file_name, minio_endpoint): def create_data_source( self, df: pd.DataFrame, - destination_name: Optional[str] = None, - suffix: Optional[str] = None, - timestamp_field="ts", + destination_name: str, + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: filename = f"{destination_name}.parquet" port = self.minio.get_exposed_port("9000") diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index dfe8e3d33bf..5a4e3f10854 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -42,16 +42,17 @@ def __init__(self, project_name: str, *args, **kwargs): iam_role=os.getenv( "AWS_IAM_ROLE", "arn:aws:iam::402087665549:role/redshift_s3_access_role" ), + workgroup="", ) def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py index c14780da97d..1481b11a106 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py @@ -48,10 +48,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 5938a0c936e..9bb8aae77f3 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -14,6 +14,7 @@ StreamFeatureView, ) from feast.data_source import DataSource, RequestSource +from feast.feature_view_projection import FeatureViewProjection from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 from tests.integration.feature_repos.universal.entities import ( customer, @@ -55,7 +56,7 @@ def conv_rate_plus_100(features_df: pd.DataFrame) -> pd.DataFrame: def conv_rate_plus_100_feature_view( - sources: Dict[str, Union[RequestSource, FeatureView]], + sources: List[Union[FeatureView, RequestSource, FeatureViewProjection]], infer_features: bool = False, features: Optional[List[Field]] = None, ) -> OnDemandFeatureView: diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py index c3872ea697f..e4e78a552f5 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from feast.repo_config import FeastConfigBaseModel @@ -7,8 +7,10 @@ class OnlineStoreCreator(ABC): def __init__(self, project_name: str, **kwargs): self.project_name = project_name + @abstractmethod def create_online_store(self) -> FeastConfigBaseModel: - ... + raise NotImplementedError + @abstractmethod def teardown(self): - ... + raise NotImplementedError diff --git a/sdk/python/tests/unit/cli/test_cli_chdir.py b/sdk/python/tests/unit/cli/test_cli_chdir.py index cf1d0312272..02e93fb0cc0 100644 --- a/sdk/python/tests/unit/cli/test_cli_chdir.py +++ b/sdk/python/tests/unit/cli/test_cli_chdir.py @@ -18,14 +18,16 @@ def test_cli_chdir() -> None: repo_path = temp_path / "my_project" / "feature_repo" assert result.returncode == 0 - result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_path) + result = runner.run(["--chdir", str(repo_path), "apply"], cwd=temp_path) assert result.returncode == 0 - result = runner.run(["--chdir", repo_path, "entities", "list"], cwd=temp_path) + result = runner.run( + ["--chdir", str(repo_path), "entities", "list"], cwd=temp_path + ) assert result.returncode == 0 result = runner.run( - ["--chdir", repo_path, "feature-views", "list"], cwd=temp_path + ["--chdir", str(repo_path), "feature-views", "list"], cwd=temp_path ) assert result.returncode == 0 @@ -34,7 +36,7 @@ def test_cli_chdir() -> None: result = runner.run( [ "--chdir", - repo_path, + str(repo_path), "materialize", start_date.isoformat(), end_date.isoformat(), @@ -44,13 +46,18 @@ def test_cli_chdir() -> None: assert result.returncode == 0 result = runner.run( - ["--chdir", repo_path, "materialize-incremental", end_date.isoformat()], + [ + "--chdir", + str(repo_path), + "materialize-incremental", + end_date.isoformat(), + ], cwd=temp_path, ) assert result.returncode == 0 - result = runner.run(["--chdir", repo_path, "registry-dump"], cwd=temp_path) + result = runner.run(["--chdir", str(repo_path), "registry-dump"], cwd=temp_path) assert result.returncode == 0 - result = runner.run(["--chdir", repo_path, "teardown"], cwd=temp_path) + result = runner.run(["--chdir", str(repo_path), "teardown"], cwd=temp_path) assert result.returncode == 0 diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index ef0cce04707..f93237fce5e 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -39,6 +39,9 @@ class MockRetrievalJob(RetrievalJob): + def to_sql(self) -> str: + return "" + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: """ Synchronously executes the underlying query and returns the result as a pandas dataframe. @@ -46,7 +49,7 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: Does not handle on demand transformations or dataset validation. For either of those, `to_df` should be used. """ - pass + return pd.DataFrame() def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: """ @@ -55,17 +58,17 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: Does not handle on demand transformations or dataset validation. For either of those, `to_arrow` should be used. """ - pass + return pyarrow.Table() @property - def full_feature_names(self) -> bool: + def full_feature_names(self) -> bool: # type: ignore """Returns True if full feature names should be applied to the results of the query.""" - pass + return False @property - def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: # type: ignore """Returns a list containing all the on demand feature views to be handled.""" - pass + return [] def persist( self, @@ -87,7 +90,7 @@ def persist( @property def metadata(self) -> Optional[RetrievalMetadata]: """Returns metadata about the retrieval job.""" - pass + raise NotImplementedError # Since RetreivalJob are not really tested for subclasses we add some tests here. @@ -115,6 +118,7 @@ def retrieval_job(request, environment): database="feast", s3_staging_location="s3://feast-integration-tests/redshift/tests/ingestion", iam_role="arn:aws:iam::402087665549:role/redshift_s3_access_role", + workgroup="", ) environment.test_repo_config.offline_store = offline_store_config return RedshiftRetrievalJob( @@ -208,7 +212,7 @@ def retrieval_job(request, environment): def test_to_sql(): - assert MockRetrievalJob().to_sql() is None + assert MockRetrievalJob().to_sql() == "" @pytest.mark.parametrize("timeout", (None, 30)) diff --git a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py index 049977489b9..48ee99e89ff 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py @@ -31,6 +31,7 @@ def test_offline_write_batch( user="user", iam_role="abcdef", s3_staging_location="s3://bucket/path", + workgroup="", ), ) diff --git a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py index 42229f8683f..ca4ed6472b9 100644 --- a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py +++ b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py @@ -45,8 +45,7 @@ def test_nullable_online_store_aws(): entity_key_serialization_version: 2 """ ), - expect_error="__root__ -> offline_store -> __root__\n" - " please specify either cluster_id & user if using provisioned clusters, or workgroup if using serverless (type=value_error)", + expect_error="4 validation errors for RepoConfig\nregion\n Field required", ) @@ -154,8 +153,7 @@ def test_extra_field(): path: "online_store.db" """ ), - expect_error="__root__ -> online_store -> that_field_should_not_be_here\n" - " extra fields not permitted (type=value_error.extra)", + expect_error="1 validation error for RepoConfig\nthat_field_should_not_be_here\n Extra inputs are not permitted", ) @@ -186,7 +184,7 @@ def test_bad_type(): path: 100500 """ ), - expect_error="__root__ -> online_store -> path\n str type expected", + expect_error="1 validation error for RepoConfig\npath\n Input should be a valid string", ) @@ -201,9 +199,7 @@ def test_no_project(): entity_key_serialization_version: 2 """ ), - expect_error="1 validation error for RepoConfig\n" - "project\n" - " field required (type=value_error.missing)", + expect_error="1 validation error for RepoConfig\nproject\n Field required", ) diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index bacc8c17206..d8c769f12c9 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -193,7 +193,7 @@ def make_feature_store_yaml( repo_path=str(Path(repo_dir_name)), entity_key_serialization_version=2, ) - config_dict = config.dict() + config_dict = config.model_dump(by_alias=True) if ( isinstance(config_dict["online_store"], dict) and "redis_type" in config_dict["online_store"] diff --git a/setup.py b/setup.py index 4fb80871b22..01e06109e09 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ "protobuf<4.23.4,>3.20", "proto-plus>=1.20.0,<2", "pyarrow>=4", - "pydantic>=1,<2", + "pydantic>=2.0.0", "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", @@ -71,7 +71,7 @@ "toml>=0.10.0,<1", "tqdm>=4,<5", "typeguard==2.13.3", - "fastapi>=0.68.0,<0.100", + "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn", "dask>=2021.1.0", @@ -126,7 +126,7 @@ "cassandra-driver>=3.24.0,<4", ] -GE_REQUIRED = ["great_expectations>=0.15.41,<0.16.0"] +GE_REQUIRED = ["great_expectations>=0.15.41"] AZURE_REQUIRED = [ "azure-storage-blob>=0.37.0", @@ -156,7 +156,7 @@ "minio==7.1.0", "mock==2.0.0", "moto<5", - "mypy>=0.981,<0.990", + "mypy>=1.4.1", "avro==1.10.0", "fsspec<2023.10.0", "urllib3>=1.25.4,<3",