From 408f4002182c15558a0dd1f123fede5841ae01a3 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Wed, 27 Mar 2024 00:56:08 -0400 Subject: [PATCH 01/20] Upgrading the python version to 3.11 and also removing the bytewax dependencies. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- .../fork_pr_integration_tests_aws.yml | 12 +- .../fork_pr_integration_tests_gcp.yml | 2 +- .../fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/build_wheels.yml | 6 +- .github/workflows/java_master_only.yml | 6 +- .github/workflows/java_pr.yml | 8 +- .github/workflows/linter.yml | 4 +- .github/workflows/master_only.yml | 2 +- .github/workflows/nightly-ci.yml | 14 +- .github/workflows/pr_integration_tests.yml | 12 +- .../workflows/pr_local_integration_tests.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- Makefile | 2 +- .../docker-compose/feast10/Dockerfile | 2 +- .../feature_servers/gcp_cloudrun/Dockerfile | 2 +- .../feature_servers/multicloud/Dockerfile | 2 +- .../feature_servers/multicloud/Dockerfile.dev | 2 +- .../contrib/bytewax/Dockerfile | 2 +- .../contrib/bytewax/__init__.py | 2 - .../bytewax_materialization_dataflow.py | 3 - .../bytewax/bytewax_materialization_engine.py | 362 +------ .../bytewax/bytewax_materialization_job.py | 63 -- .../infra/transformation_servers/Dockerfile | 2 +- .../requirements/py3.11-ci-requirements.txt | 993 ++++++++++++++++++ .../requirements/py3.11-requirements.txt | 202 ++++ setup.py | 7 +- 27 files changed, 1245 insertions(+), 475 deletions(-) delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py create mode 100644 sdk/python/requirements/py3.11-ci-requirements.txt create mode 100644 sdk/python/requirements/py3.11-requirements.txt diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index 49fd16ef5d4..4a2dba6c2dc 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -33,21 +33,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v2 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) if: steps.cache-primes.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) if: steps.cache-primes.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -67,7 +67,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index cf85cc8873f..3b448765043 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index 0b1c8a48ced..d93df465688 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f0851f5bb04..3d9dbe33aa6 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -59,7 +59,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Setup Node uses: actions/setup-node@v3 @@ -86,7 +86,7 @@ jobs: id: setup-python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" architecture: x64 - name: Setup Node uses: actions/setup-node@v3 @@ -137,7 +137,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest ] - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] from-source: [ True, False ] env: # this script is for testing servers diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 79b456e571e..ba74d993672 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -25,7 +25,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' @@ -95,7 +95,7 @@ jobs: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest env: - PYTHON: 3.9 + PYTHON: 3.11 steps: - uses: actions/checkout@v3 with: @@ -110,7 +110,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Get pip cache dir id: pip-cache diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index b78b5297d22..c243814c202 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -88,7 +88,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' @@ -111,7 +111,7 @@ jobs: runs-on: ubuntu-latest needs: unit-test-java env: - PYTHON: 3.9 + PYTHON: 3.11 steps: - uses: actions/checkout@v3 with: @@ -128,7 +128,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: '3.11' architecture: 'x64' - uses: actions/cache@v2 with: @@ -158,7 +158,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Get pip cache dir id: pip-cache diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a1747db1356..8c1332493c8 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -6,14 +6,14 @@ jobs: lint-python: runs-on: [ubuntu-latest] env: - PYTHON: 3.9 + PYTHON: 3.11 steps: - uses: actions/checkout@v3 - name: Setup Python id: setup-python uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Get pip cache dir id: pip-cache diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index ded5310b50c..ac07b29677f 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -65,7 +65,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10" ] + python-version: ["3.9", "3.10", "3.11"] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index e39f2e1c007..6f444b39999 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -36,7 +36,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Set up AWS SDK uses: aws-actions/configure-aws-credentials@v1 @@ -89,21 +89,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v2 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) if: steps.cache-primes.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) if: steps.cache-primes.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -124,7 +124,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index b335c0f042f..c601dd4e4f3 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -47,21 +47,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v2 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) if: steps.cache-primes.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) if: steps.cache-primes.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -86,7 +86,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10" ] + python-version: [ "3.9", "3.10", "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 17ff54b1f84..d92a6051eea 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10" ] + python-version: [ "3.9", "3.10", "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 432ab4bb585..94f72500e90 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -160,7 +160,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: '3.11' architecture: 'x64' - uses: actions/cache@v2 with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7e2e3b577af..51bb9b74e20 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10" ] + python-version: [ "3.9", "3.10", "3.11"] os: [ ubuntu-latest, macOS-latest ] exclude: - os: macOS-latest diff --git a/Makefile b/Makefile index 1a1d3b6e920..59324255292 100644 --- a/Makefile +++ b/Makefile @@ -366,7 +366,7 @@ kill-trino-locally: cd ${ROOT_DIR}; docker stop trino install-protoc-dependencies: - pip install --ignore-installed protobuf==4.23.4 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0 + pip install --ignore-installed protobuf==4.24.0 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0 install-feast-ci-locally: pip install -e ".[ci]" diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index 7e36658caef..a4e2a0dfc20 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 WORKDIR /usr/src/ diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile index 6e3ff424eab..6b89d4f73c1 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim RUN apt-get update && apt-get install -y git diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 5d2d425cda4..6461e095169 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 # Input the feast version to install # This requires feast package to be available in pypi before building this image diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 900578f55db..a161acf8f3b 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 # Input the feast version to install # This requires feast package to be available in pypi before building this image diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile index a7d0af9b416..a785604018a 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-bullseye AS build +FROM python:3.11-slim-bullseye AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes git diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py b/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py index 0838a4c0d59..e108c03d801 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py @@ -3,12 +3,10 @@ BytewaxMaterializationEngine, BytewaxMaterializationEngineConfig, ) -from .bytewax_materialization_job import BytewaxMaterializationJob from .bytewax_materialization_task import BytewaxMaterializationTask __all__ = [ "BytewaxMaterializationTask", - "BytewaxMaterializationJob", "BytewaxMaterializationDataflow", "BytewaxMaterializationEngine", "BytewaxMaterializationEngineConfig", diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py index 6fc53b67f2f..17cd8c3b805 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py @@ -5,9 +5,6 @@ import pyarrow as pa import pyarrow.parquet as pq from bytewax.dataflow import Dataflow # type: ignore -from bytewax.execution import cluster_main -from bytewax.inputs import ManualInputConfig -from bytewax.outputs import ManualOutputConfig from feast import FeatureStore, FeatureView, RepoConfig from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index 060a47ce585..05d3f87af24 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -5,11 +5,6 @@ from typing import Callable, List, Literal, Sequence, Union import yaml -from kubernetes import client -from kubernetes import config as k8s_config -from kubernetes import utils -from kubernetes.client.exceptions import ApiException -from kubernetes.utils import FailToCreateError from pydantic import StrictStr from tqdm import tqdm @@ -29,12 +24,12 @@ from feast.stream_feature_view import StreamFeatureView from feast.utils import _get_column_names -from .bytewax_materialization_job import BytewaxMaterializationJob +from ...local_engine import LocalMaterializationEngine, LocalMaterializationEngineConfig logger = logging.getLogger(__name__) -class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): +class BytewaxMaterializationEngineConfig(LocalMaterializationEngineConfig): """Batch Materialization Engine config for Bytewax""" type: Literal["bytewax"] = "bytewax" @@ -92,7 +87,7 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): """(optional) Print pod logs on job failure. Only applies to synchronous materialization""" -class BytewaxMaterializationEngine(BatchMaterializationEngine): +class BytewaxMaterializationEngine(LocalMaterializationEngine): def __init__( self, *, @@ -111,11 +106,6 @@ def __init__( self.offline_store = offline_store self.online_store = online_store - k8s_config.load_config() - - self.k8s_client = client.api_client.ApiClient() - self.v1 = client.CoreV1Api(self.k8s_client) - self.batch_v1 = client.BatchV1Api(self.k8s_client) self.batch_engine_config = repo_config.batch_engine self.namespace = self.batch_engine_config.namespace @@ -149,348 +139,4 @@ def materialize( registry: BaseRegistry, tasks: List[MaterializationTask], ) -> List[MaterializationJob]: - return [ - self._materialize_one( - registry, - task.feature_view, - task.start_time, - task.end_time, - task.project, - task.tqdm_builder, - ) - for task in tasks - ] - - def _materialize_one( - self, - registry: BaseRegistry, - feature_view: Union[BatchFeatureView, StreamFeatureView, FeatureView], - start_date: datetime, - end_date: datetime, - project: str, - tqdm_builder: Callable[[int], tqdm], - ): - entities = [] - for entity_name in feature_view.entities: - entities.append(registry.get_entity(entity_name, project)) - - ( - join_key_columns, - feature_name_columns, - timestamp_field, - created_timestamp_column, - ) = _get_column_names(feature_view, entities) - - offline_job = self.offline_store.pull_latest_from_table_or_query( - config=self.repo_config, - data_source=feature_view.batch_source, - join_key_columns=join_key_columns, - feature_name_columns=feature_name_columns, - timestamp_field=timestamp_field, - created_timestamp_column=created_timestamp_column, - start_date=start_date, - end_date=end_date, - ) - - paths = offline_job.to_remote_storage() - if self.batch_engine_config.synchronous: - offset = 0 - total_pods = len(paths) - batch_size = self.batch_engine_config.job_batch_size - if batch_size < 1: - raise ValueError("job_batch_size must be a value greater than 0") - if batch_size < self.batch_engine_config.max_parallelism: - logger.warning( - "job_batch_size is less than max_parallelism. Setting job_batch_size = max_parallelism" - ) - batch_size = self.batch_engine_config.max_parallelism - - while True: - next_offset = min(offset + batch_size, total_pods) - job = self._await_path_materialization( - paths[offset:next_offset], - feature_view, - offset, - next_offset, - total_pods, - ) - offset += batch_size - if ( - offset >= total_pods - or job.status() == MaterializationJobStatus.ERROR - ): - break - else: - job_id = str(uuid.uuid4()) - job = self._create_kubernetes_job(job_id, paths, feature_view) - - return job - - def _await_path_materialization( - self, paths, feature_view, batch_start, batch_end, total_pods - ): - job_id = str(uuid.uuid4()) - job = self._create_kubernetes_job(job_id, paths, feature_view) - - try: - while job.status() in ( - MaterializationJobStatus.WAITING, - MaterializationJobStatus.RUNNING, - ): - logger.info( - f"{feature_view.name} materialization for pods {batch_start}-{batch_end} " - f"(of {total_pods}) running..." - ) - sleep(30) - logger.info( - f"{feature_view.name} materialization for pods {batch_start}-{batch_end} " - f"(of {total_pods}) complete with status {job.status()}" - ) - except BaseException as e: - logger.info(f"Deleting job {job.job_id()}") - try: - self.batch_v1.delete_namespaced_job(job.job_id(), self.namespace) - except ApiException as ae: - logger.warning(f"Could not delete job due to API Error: {ae.body}") - raise e - finally: - logger.info(f"Deleting configmap {self._configmap_name(job_id)}") - try: - self.v1.delete_namespaced_config_map( - self._configmap_name(job_id), self.namespace - ) - except ApiException as ae: - logger.warning( - f"Could not delete configmap due to API Error: {ae.body}" - ) - - if ( - job.status() == MaterializationJobStatus.ERROR - and self.batch_engine_config.print_pod_logs_on_failure - ): - self._print_pod_logs(job.job_id(), feature_view, batch_start) - - return job - - def _print_pod_logs(self, job_id, feature_view, offset=0): - pods_list = self.v1.list_namespaced_pod( - namespace=self.namespace, - label_selector=f"job-name={job_id}", - ).items - for i, pod in enumerate(pods_list): - logger.info(f"Logging output for {feature_view.name} pod {offset+i}") - try: - logger.info( - self.v1.read_namespaced_pod_log(pod.metadata.name, self.namespace) - ) - except ApiException as e: - logger.warning(f"Could not retrieve pod logs due to: {e.body}") - - def _create_kubernetes_job(self, job_id, paths, feature_view): - try: - # Create a k8s configmap with information needed by bytewax - self._create_configuration_map(job_id, paths, feature_view, self.namespace) - - # Create the k8s job definition - self._create_job_definition( - job_id, - self.namespace, - len(paths), # Create a pod for each parquet file - self.batch_engine_config.env, - ) - logger.info( - f"Created job `dataflow-{job_id}` on namespace `{self.namespace}`" - ) - except FailToCreateError as failures: - return BytewaxMaterializationJob(job_id, self.namespace, error=failures) - - return BytewaxMaterializationJob(job_id, self.namespace) - - def _create_configuration_map(self, job_id, paths, feature_view, namespace): - """Create a Kubernetes configmap for this job""" - - feature_store_configuration = yaml.dump(self.repo_config.dict()) - - materialization_config = yaml.dump( - {"paths": paths, "feature_view": feature_view.name} - ) - - labels = {"feast-bytewax-materializer": "configmap"} - configmap_manifest = { - "kind": "ConfigMap", - "apiVersion": "v1", - "metadata": { - "name": self._configmap_name(job_id), - "labels": {**labels, **self.batch_engine_config.labels}, - }, - "data": { - "feature_store.yaml": feature_store_configuration, - "bytewax_materialization_config.yaml": materialization_config, - }, - } - self.v1.create_namespaced_config_map( - namespace=namespace, - body=configmap_manifest, - ) - - def _configmap_name(self, job_id): - return f"feast-{job_id}" - - def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0): - """Create a kubernetes job definition.""" - job_env = [ - {"name": "RUST_BACKTRACE", "value": "full"}, - { - "name": "BYTEWAX_PYTHON_FILE_PATH", - "value": "/bytewax/dataflow.py", - }, - {"name": "BYTEWAX_WORKDIR", "value": "/bytewax"}, - { - "name": "BYTEWAX_WORKERS_PER_PROCESS", - "value": "1", - }, - { - "name": "BYTEWAX_POD_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.annotations['batch.kubernetes.io/job-completion-index']", - } - }, - }, - { - "name": "BYTEWAX_REPLICAS", - "value": "1", - }, - { - "name": "BYTEWAX_KEEP_CONTAINER_ALIVE", - "value": "false", - }, - { - "name": "BYTEWAX_STATEFULSET_NAME", - "value": f"dataflow-{job_id}", - }, - { - "name": "BYTEWAX_MINI_BATCH_SIZE", - "value": str(self.batch_engine_config.mini_batch_size), - }, - ] - # Add any Feast configured environment variables - job_env.extend(env) - - securityContextCapabilities = None - if self.batch_engine_config.include_security_context_capabilities: - securityContextCapabilities = { - "add": ["NET_BIND_SERVICE"], - "drop": ["ALL"], - } - - job_labels = {"feast-bytewax-materializer": "job"} - pod_labels = {"feast-bytewax-materializer": "pod"} - job_definition = { - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": f"dataflow-{job_id}", - "namespace": namespace, - "labels": {**job_labels, **self.batch_engine_config.labels}, - }, - "spec": { - "ttlSecondsAfterFinished": 3600, - "backoffLimit": self.batch_engine_config.retry_limit, - "completions": pods, - "parallelism": min(pods, self.batch_engine_config.max_parallelism), - "activeDeadlineSeconds": self.batch_engine_config.active_deadline_seconds, - "completionMode": "Indexed", - "template": { - "metadata": { - "annotations": self.batch_engine_config.annotations, - "labels": {**pod_labels, **self.batch_engine_config.labels}, - }, - "spec": { - "restartPolicy": "Never", - "subdomain": f"dataflow-{job_id}", - "imagePullSecrets": self.batch_engine_config.image_pull_secrets, - "serviceAccountName": self.batch_engine_config.service_account_name, - "initContainers": [ - { - "env": [ - { - "name": "BYTEWAX_REPLICAS", - "value": f"{pods}", - } - ], - "image": "busybox", - "imagePullPolicy": "IfNotPresent", - "name": "init-hostfile", - "resources": {}, - "securityContext": { - "allowPrivilegeEscalation": False, - "capabilities": securityContextCapabilities, - "readOnlyRootFilesystem": True, - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - {"mountPath": "/etc/bytewax", "name": "hostfile"}, - { - "mountPath": "/tmp/bytewax/", - "name": "python-files", - }, - { - "mountPath": "/var/feast/", - "name": self._configmap_name(job_id), - }, - ], - } - ], - "containers": [ - { - "command": ["sh", "-c", "sh ./entrypoint.sh"], - "env": job_env, - "image": self.batch_engine_config.image, - "imagePullPolicy": "Always", - "name": "process", - "ports": [ - { - "containerPort": 9999, - "name": "process", - "protocol": "TCP", - } - ], - "resources": self.batch_engine_config.resources, - "securityContext": { - "allowPrivilegeEscalation": False, - "capabilities": securityContextCapabilities, - "readOnlyRootFilesystem": False, - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - {"mountPath": "/etc/bytewax", "name": "hostfile"}, - { - "mountPath": "/var/feast/", - "name": self._configmap_name(job_id), - }, - ], - } - ], - "volumes": [ - {"emptyDir": {}, "name": "hostfile"}, - { - "configMap": { - "defaultMode": 420, - "name": self._configmap_name(job_id), - }, - "name": "python-files", - }, - { - "configMap": {"name": self._configmap_name(job_id)}, - "name": self._configmap_name(job_id), - }, - ], - }, - }, - }, - } - utils.create_from_dict(self.k8s_client, job_definition) + super().materialize(self, registry, tasks) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py deleted file mode 100644 index da969d5a880..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import Optional - -from kubernetes import client - -from feast.infra.materialization.batch_materialization_engine import ( - MaterializationJob, - MaterializationJobStatus, -) - - -class BytewaxMaterializationJob(MaterializationJob): - def __init__( - self, - job_id, - namespace, - error: Optional[BaseException] = None, - ): - super().__init__() - self._job_id = job_id - self.namespace = namespace - self._error: Optional[BaseException] = error - self.batch_v1 = client.BatchV1Api() - - def error(self): - return self._error - - def status(self): - if self._error is not None: - return MaterializationJobStatus.ERROR - else: - # TODO: Find a better way to parse status? - job_status = self.batch_v1.read_namespaced_job_status( - self.job_id(), self.namespace - ).status - if job_status.active is not None: - if job_status.completion_time is None: - return MaterializationJobStatus.RUNNING - else: - if ( - job_status.completion_time is not None - and job_status.conditions[0].type == "Complete" - ): - return MaterializationJobStatus.SUCCEEDED - - if ( - job_status.conditions is not None - and job_status.conditions[0].type == "Failed" - ): - self._error = Exception( - f"Job {self.job_id()} failed with reason: " - f"{job_status.conditions[0].message}" - ) - return MaterializationJobStatus.ERROR - return MaterializationJobStatus.WAITING - - def should_be_retried(self): - return False - - def job_id(self): - return f"dataflow-{self._job_id}" - - def url(self): - return None diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile index 41f272c757c..cd46b0baa90 100644 --- a/sdk/python/feast/infra/transformation_servers/Dockerfile +++ b/sdk/python/feast/infra/transformation_servers/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim RUN apt-get update && apt-get install -y git diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt new file mode 100644 index 00000000000..6b724b8136a --- /dev/null +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -0,0 +1,993 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.11-ci-requirements.txt +# +alabaster==0.7.16 + # via sphinx +altair==4.2.2 + # via great-expectations +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # httpx + # jupyter-server + # starlette + # watchfiles +appdirs==1.4.4 + # via fissix +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asn1crypto==1.5.1 + # via snowflake-connector-python +assertpy==1.1 + # via feast (setup.py) +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +atpublic==4.0 + # via ibis-framework +attrs==23.2.0 + # via + # bowler + # jsonschema + # referencing +avro==1.10.0 + # via feast (setup.py) +azure-core==1.30.1 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.15.0 + # via feast (setup.py) +azure-storage-blob==12.19.1 + # via feast (setup.py) +babel==2.14.0 + # via + # jupyterlab-server + # sphinx +beautifulsoup4==4.12.3 + # via nbconvert +bidict==0.23.1 + # via ibis-framework +black==22.12.0 + # via feast (setup.py) +bleach==6.1.0 + # via nbconvert +boto3==1.34.71 + # via + # feast (setup.py) + # moto +botocore==1.34.71 + # via + # boto3 + # moto + # s3transfer +bowler==0.9.0 + # via feast (setup.py) +build==1.1.1 + # via + # feast (setup.py) + # pip-tools +cachecontrol==0.14.0 + # via firebase-admin +cachetools==5.3.3 + # via google-auth +cassandra-driver==3.29.1 + # via feast (setup.py) +certifi==2024.2.2 + # via + # httpcore + # httpx + # minio + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # argon2-cffi-bindings + # cryptography + # snowflake-connector-python +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # black + # bowler + # dask + # feast (setup.py) + # geomet + # great-expectations + # moreorless + # pip-tools + # uvicorn +cloudpickle==3.0.0 + # via dask +colorama==0.4.6 + # via + # feast (setup.py) + # great-expectations +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.4.4 + # via pytest-cov +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # feast (setup.py) + # great-expectations + # moto + # msal + # pyjwt + # pyopenssl + # snowflake-connector-python + # types-pyopenssl + # types-redis +dask==2024.2.1 + # via feast (setup.py) +db-dtypes==1.2.0 + # via google-cloud-bigquery +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +deprecation==2.1.0 + # via testcontainers +dill==0.3.8 + # via feast (setup.py) +distlib==0.3.8 + # via virtualenv +docker==7.0.0 + # via + # feast (setup.py) + # testcontainers +docutils==0.19 + # via sphinx +entrypoints==0.4 + # via altair +execnet==2.0.2 + # via pytest-xdist +executing==2.0.1 + # via stack-data +fastapi==0.110.0 + # via feast (setup.py) +fastjsonschema==2.19.1 + # via nbformat +filelock==3.13.3 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 + # via feast (setup.py) +fissix==21.11.13 + # via bowler +flake8==6.0.0 + # via feast (setup.py) +fqdn==1.5.1 + # via jsonschema +fsspec==2023.12.2 + # via + # dask + # feast (setup.py) +geojson==2.5.0 + # via rockset +geomet==0.2.1.post1 + # via cassandra-driver +google-api-core[grpc]==2.18.0 + # via + # feast (setup.py) + # firebase-admin + # google-api-python-client + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-api-python-client==2.123.0 + # via firebase-admin +google-auth==2.29.0 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-core + # google-cloud-storage +google-auth-httplib2==0.2.0 + # via google-api-python-client +google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) +google-cloud-bigquery-storage==2.24.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.0 + # via feast (setup.py) +google-cloud-core==2.4.1 + # via + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-cloud-datastore==2.19.0 + # via feast (setup.py) +google-cloud-firestore==2.15.0 + # via firebase-admin +google-cloud-storage==2.16.0 + # via + # feast (setup.py) + # firebase-admin +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.63.0 + # via + # feast (setup.py) + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +great-expectations==0.18.12 + # via feast (setup.py) +greenlet==3.0.3 + # via sqlalchemy +grpc-google-iam-v1==0.13.0 + # via google-cloud-bigtable +grpcio==1.62.1 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools +grpcio-health-checking==1.62.1 + # via feast (setup.py) +grpcio-reflection==1.62.1 + # via feast (setup.py) +grpcio-status==1.62.1 + # via google-api-core +grpcio-testing==1.62.1 + # via feast (setup.py) +grpcio-tools==1.62.1 + # via feast (setup.py) +gunicorn==21.2.0 + # via feast (setup.py) +h11==0.14.0 + # via + # httpcore + # uvicorn +happybase==1.2.0 + # via feast (setup.py) +hazelcast-python-client==5.3.0 + # via feast (setup.py) +hiredis==2.3.2 + # via feast (setup.py) +httpcore==1.0.4 + # via httpx +httplib2==0.22.0 + # via + # google-api-python-client + # google-auth-httplib2 +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via + # feast (setup.py) + # jupyterlab +ibis==3.3.0 + # via feast (setup.py) +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 + # via pre-commit +idna==3.6 + # via + # anyio + # httpx + # jsonschema + # requests + # snowflake-connector-python +imagesize==1.4.1 + # via sphinx +importlib-metadata==6.11.0 + # via + # dask + # feast (setup.py) +importlib-resources==6.4.0 + # via feast (setup.py) +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.3 + # via jupyterlab +ipython==8.22.2 + # via + # great-expectations + # ipykernel + # ipywidgets +ipywidgets==8.1.2 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +isort==5.13.2 + # via feast (setup.py) +jedi==0.19.1 + # via ipython +jinja2==3.1.3 + # via + # altair + # feast (setup.py) + # great-expectations + # jupyter-server + # jupyterlab + # jupyterlab-server + # moto + # nbconvert + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +json5==0.9.24 + # via jupyterlab-server +jsonpatch==1.33 + # via great-expectations +jsonpointer==2.4 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.21.1 + # via + # altair + # feast (setup.py) + # great-expectations + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter-client==8.6.1 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.4 + # via jupyterlab +jupyter-server==2.13.0 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.1.5 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.25.4 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.10 + # via ipywidgets +locket==1.0.0 + # via partd +makefun==1.15.2 + # via great-expectations +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.21.1 + # via great-expectations +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mccabe==0.7.0 + # via flake8 +mdurl==0.1.2 + # via markdown-it-py +minio==7.1.0 + # via feast (setup.py) +mistune==3.0.2 + # via + # great-expectations + # nbconvert +mmh3==4.1.0 + # via feast (setup.py) +mock==2.0.0 + # via feast (setup.py) +moreorless==0.4.0 + # via bowler +moto==4.2.14 + # via feast (setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +msgpack==1.0.8 + # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework +mypy==1.9.0 + # via + # feast (setup.py) + # sqlalchemy +mypy-extensions==1.0.0 + # via + # black + # mypy +mypy-protobuf==3.3.0 + # via feast (setup.py) +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.3 + # via jupyter-server +nbformat==5.10.3 + # via + # great-expectations + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.8.0 + # via pre-commit +notebook==7.1.2 + # via great-expectations +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook +numpy==1.24.4 + # via + # altair + # db-dtypes + # feast (setup.py) + # great-expectations + # ibis-framework + # pandas + # pyarrow + # scipy +overrides==7.7.0 + # via jupyter-server +packaging==24.0 + # via + # build + # dask + # db-dtypes + # deprecation + # docker + # google-cloud-bigquery + # great-expectations + # gunicorn + # ibis-substrait + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # msal-extensions + # nbconvert + # pytest + # snowflake-connector-python + # sphinx +pandas==2.2.1 + # via + # altair + # db-dtypes + # feast (setup.py) + # google-cloud-bigquery + # great-expectations + # ibis-framework + # snowflake-connector-python +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.3 + # via jedi +parsy==2.1 + # via ibis-framework +partd==1.4.1 + # via dask +pathspec==0.12.1 + # via black +pbr==6.0.0 + # via mock +pexpect==4.9.0 + # via ipython +pip-tools==7.4.1 + # via feast (setup.py) +platformdirs==3.11.0 + # via + # black + # jupyter-core + # snowflake-connector-python + # virtualenv +pluggy==1.4.0 + # via pytest +ply==3.11 + # via thriftpy2 +portalocker==2.8.2 + # via msal-extensions +pre-commit==3.3.1 + # via feast (setup.py) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.43 + # via ipython +proto-plus==1.23.0 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore +protobuf==4.25.3 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools + # mypy-protobuf + # proto-plus + # substrait +psutil==5.9.0 + # via + # feast (setup.py) + # ipykernel +psycopg2-binary==2.9.9 + # via feast (setup.py) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via feast (setup.py) +py-cpuinfo==9.0.0 + # via pytest-benchmark +py4j==0.10.9.7 + # via pyspark +pyarrow==15.0.2 + # via + # db-dtypes + # feast (setup.py) + # google-cloud-bigquery + # ibis-framework + # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework +pyasn1==0.5.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pybindgen==0.22.1 + # via feast (setup.py) +pycodestyle==2.10.0 + # via flake8 +pycparser==2.21 + # via cffi +pydantic==2.6.4 + # via + # fastapi + # feast (setup.py) + # great-expectations +pydantic-core==2.16.3 + # via pydantic +pyflakes==3.0.1 + # via flake8 +pygments==2.17.2 + # via + # feast (setup.py) + # ipython + # nbconvert + # rich + # sphinx +pyjwt[crypto]==2.8.0 + # via + # msal + # snowflake-connector-python +pymssql==2.2.11 + # via feast (setup.py) +pymysql==1.1.0 + # via feast (setup.py) +pyodbc==5.1.0 + # via feast (setup.py) +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via + # great-expectations + # httplib2 +pyproject-hooks==1.0.0 + # via + # build + # pip-tools +pyspark==3.5.1 + # via feast (setup.py) +pytest==7.4.4 + # via + # feast (setup.py) + # pytest-benchmark + # pytest-cov + # pytest-lazy-fixture + # pytest-mock + # pytest-ordering + # pytest-timeout + # pytest-xdist +pytest-benchmark==3.4.1 + # via feast (setup.py) +pytest-cov==5.0.0 + # via feast (setup.py) +pytest-lazy-fixture==0.6.3 + # via feast (setup.py) +pytest-mock==1.10.4 + # via feast (setup.py) +pytest-ordering==0.6 + # via feast (setup.py) +pytest-timeout==1.4.2 + # via feast (setup.py) +pytest-xdist==3.5.0 + # via feast (setup.py) +python-dateutil==2.9.0.post0 + # via + # arrow + # botocore + # google-cloud-bigquery + # great-expectations + # ibis-framework + # jupyter-client + # moto + # pandas + # rockset + # trino +python-dotenv==1.0.1 + # via uvicorn +python-json-logger==2.0.7 + # via jupyter-events +pytz==2024.1 + # via + # great-expectations + # ibis-framework + # pandas + # snowflake-connector-python + # trino +pyyaml==6.0.1 + # via + # dask + # feast (setup.py) + # ibis-substrait + # jupyter-events + # pre-commit + # responses + # uvicorn +pyzmq==25.1.2 + # via + # ipykernel + # jupyter-client + # jupyter-server +redis==4.6.0 + # via feast (setup.py) +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2023.12.25 + # via feast (setup.py) +requests==2.31.0 + # via + # azure-core + # cachecontrol + # docker + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # great-expectations + # jupyterlab-server + # moto + # msal + # responses + # snowflake-connector-python + # sphinx + # trino +responses==0.25.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rich==13.7.1 + # via ibis-framework +rockset==2.1.1 + # via feast (setup.py) +rpds-py==0.18.0 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +ruamel-yaml==0.17.17 + # via great-expectations +s3transfer==0.10.1 + # via boto3 +scipy==1.12.0 + # via great-expectations +send2trash==1.8.2 + # via jupyter-server +six==1.16.0 + # via + # asttokens + # azure-core + # bleach + # geomet + # happybase + # isodate + # mock + # python-dateutil + # rfc3339-validator + # thriftpy2 +sniffio==1.3.1 + # via + # anyio + # httpx +snowballstemmer==2.2.0 + # via sphinx +snowflake-connector-python[pandas]==3.7.1 + # via feast (setup.py) +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sphinx==6.2.1 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy[mypy]==1.4.52 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework +stack-data==0.6.3 + # via ipython +starlette==0.36.3 + # via fastapi +substrait==0.15.0 + # via ibis-substrait +tabulate==0.9.0 + # via feast (setup.py) +tenacity==8.2.3 + # via feast (setup.py) +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +testcontainers==3.7.1 + # via feast (setup.py) +thriftpy2==0.4.20 + # via happybase +tinycss2==1.2.1 + # via nbconvert +toml==0.10.2 + # via feast (setup.py) +tomlkit==0.12.4 + # via snowflake-connector-python +toolz==0.12.1 + # via + # altair + # dask + # ibis-framework + # partd +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +tqdm==4.66.2 + # via + # feast (setup.py) + # great-expectations +traitlets==5.14.2 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +trino==0.328.0 + # via feast (setup.py) +typeguard==4.2.1 + # via feast (setup.py) +types-protobuf==3.19.22 + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.1 + # via feast (setup.py) +types-pyopenssl==24.0.0.20240311 + # via types-redis +types-python-dateutil==2.9.0.20240316 + # via + # arrow + # feast (setup.py) +types-pytz==2024.1.0.20240203 + # via feast (setup.py) +types-pyyaml==6.0.12.20240311 + # via feast (setup.py) +types-redis==4.6.0.20240311 + # via feast (setup.py) +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==69.2.0.20240317 + # via feast (setup.py) +types-tabulate==0.9.0.20240106 + # via feast (setup.py) +types-urllib3==1.26.25.14 + # via types-requests +typing-extensions==4.10.0 + # via + # azure-core + # azure-storage-blob + # fastapi + # great-expectations + # ibis-framework + # mypy + # pydantic + # pydantic-core + # snowflake-connector-python + # sqlalchemy2-stubs + # typeguard +tzdata==2024.1 + # via pandas +tzlocal==5.2 + # via + # great-expectations + # trino +uri-template==1.3.0 + # via jsonschema +uritemplate==4.1.1 + # via google-api-python-client +urllib3==1.26.18 + # via + # botocore + # docker + # feast (setup.py) + # great-expectations + # minio + # requests + # responses + # rockset +uvicorn[standard]==0.29.0 + # via feast (setup.py) +uvloop==0.19.0 + # via uvicorn +virtualenv==20.23.0 + # via + # feast (setup.py) + # pre-commit +volatile==2.1.0 + # via bowler +watchfiles==0.21.0 + # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.7.0 + # via jupyter-server +websockets==12.0 + # via uvicorn +werkzeug==3.0.1 + # via moto +wheel==0.43.0 + # via pip-tools +widgetsnbextension==4.0.10 + # via ipywidgets +wrapt==1.16.0 + # via testcontainers +xmltodict==0.13.0 + # via moto +zipp==3.18.1 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt new file mode 100644 index 00000000000..90d336a40aa --- /dev/null +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -0,0 +1,202 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=sdk/python/requirements/py3.11-requirements.txt +# +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # httpx + # starlette + # watchfiles +appdirs==1.4.4 + # via fissix +attrs==23.2.0 + # via + # bowler + # jsonschema + # referencing +bowler==0.9.0 + # via feast (setup.py) +certifi==2024.2.2 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # bowler + # dask + # feast (setup.py) + # moreorless + # uvicorn +cloudpickle==3.0.0 + # via dask +colorama==0.4.6 + # via feast (setup.py) +dask==2024.2.1 + # via feast (setup.py) +dill==0.3.8 + # via feast (setup.py) +fastapi==0.110.0 + # via feast (setup.py) +fissix==21.11.13 + # via bowler +fsspec==2024.3.1 + # via dask +greenlet==3.0.3 + # via sqlalchemy +gunicorn==21.2.0 + # via feast (setup.py) +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.4 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via feast (setup.py) +ibis==3.3.0 + # via feast (setup.py) +idna==3.6 + # via + # anyio + # httpx + # requests +importlib-metadata==6.11.0 + # via + # dask + # feast (setup.py) +importlib-resources==6.4.0 + # via feast (setup.py) +jinja2==3.1.3 + # via feast (setup.py) +jsonschema==4.21.1 + # via feast (setup.py) +jsonschema-specifications==2023.12.1 + # via jsonschema +locket==1.0.0 + # via partd +markupsafe==2.1.5 + # via jinja2 +mmh3==4.1.0 + # via feast (setup.py) +moreorless==0.4.0 + # via bowler +mypy==1.9.0 + # via sqlalchemy +mypy-extensions==1.0.0 + # via mypy +mypy-protobuf==3.5.0 + # via feast (setup.py) +numpy==1.24.4 + # via + # feast (setup.py) + # pandas + # pyarrow +packaging==24.0 + # via + # dask + # gunicorn +pandas==2.2.1 + # via feast (setup.py) +partd==1.4.1 + # via dask +proto-plus==1.23.0 + # via feast (setup.py) +protobuf==4.25.3 + # via + # feast (setup.py) + # mypy-protobuf + # proto-plus +pyarrow==15.0.2 + # via feast (setup.py) +pydantic==2.6.4 + # via + # fastapi + # feast (setup.py) +pydantic-core==2.16.3 + # via pydantic +pygments==2.17.2 + # via feast (setup.py) +python-dateutil==2.9.0.post0 + # via pandas +python-dotenv==1.0.1 + # via uvicorn +pytz==2024.1 + # via pandas +pyyaml==6.0.1 + # via + # dask + # feast (setup.py) + # uvicorn +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via feast (setup.py) +rpds-py==0.18.0 + # via + # jsonschema + # referencing +six==1.16.0 + # via python-dateutil +sniffio==1.3.1 + # via + # anyio + # httpx +sqlalchemy[mypy]==1.4.52 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy +starlette==0.36.3 + # via fastapi +tabulate==0.9.0 + # via feast (setup.py) +tenacity==8.2.3 + # via feast (setup.py) +toml==0.10.2 + # via feast (setup.py) +toolz==0.12.1 + # via + # dask + # partd +tqdm==4.66.2 + # via feast (setup.py) +typeguard==4.2.1 + # via feast (setup.py) +types-protobuf==4.24.0.20240311 + # via mypy-protobuf +typing-extensions==4.10.0 + # via + # fastapi + # mypy + # pydantic + # pydantic-core + # sqlalchemy2-stubs + # typeguard +tzdata==2024.1 + # via pandas +urllib3==2.2.1 + # via requests +uvicorn[standard]==0.29.0 + # via feast (setup.py) +uvloop==0.19.0 + # via uvicorn +volatile==2.1.0 + # via bowler +watchfiles==0.21.0 + # via uvicorn +websockets==12.0 + # via uvicorn +zipp==3.18.1 + # via importlib-metadata diff --git a/setup.py b/setup.py index 92d73564113..750b5c333f9 100644 --- a/setup.py +++ b/setup.py @@ -74,13 +74,14 @@ "httpx>=0.23.3", "importlib-resources>=6.0.0,<7", "importlib_metadata>=6.8.0,<7", + "ibis~=3.3.0", ] GCP_REQUIRED = [ "google-api-core>=1.23.0,<3", "googleapis-common-protos>=1.52.0,<2", "google-cloud-bigquery[pandas]>=2,<3.13.0", - "google-cloud-bigquery-storage >= 2.0.0,<3", + "google-cloud-bigquery-storage >= 2.23.0,<3", "google-cloud-datastore>=2.1.0,<3", "google-cloud-storage>=1.34.0,<3", "google-cloud-bigtable>=2.11.0,<3", @@ -94,8 +95,6 @@ AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0"] -BYTEWAX_REQUIRED = ["bytewax==0.15.1", "docker>=5.0.2", "kubernetes<=20.13.0"] - SNOWFLAKE_REQUIRED = [ "snowflake-connector-python[pandas]>=3.7,<4", ] @@ -199,7 +198,6 @@ + GCP_REQUIRED + REDIS_REQUIRED + AWS_REQUIRED - + BYTEWAX_REQUIRED + SNOWFLAKE_REQUIRED + SPARK_REQUIRED + POSTGRES_REQUIRED @@ -366,7 +364,6 @@ def run(self): "ci": CI_REQUIRED, "gcp": GCP_REQUIRED, "aws": AWS_REQUIRED, - "bytewax": BYTEWAX_REQUIRED, "redis": REDIS_REQUIRED, "snowflake": SNOWFLAKE_REQUIRED, "spark": SPARK_REQUIRED, From 1214f7186e6bd56e0c6353c7ddd72dbfb30a22ab Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Wed, 27 Mar 2024 12:11:37 -0400 Subject: [PATCH 02/20] Removing missing bytewax dependencies from the previous commit. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- .../contrib/bytewax/Dockerfile | 29 ------- .../contrib/bytewax/__init__.py | 2 - .../bytewax_materialization_dataflow.py | 80 ------------------- .../contrib/bytewax/dataflow.py | 28 ------- .../contrib/bytewax/entrypoint.sh | 4 - 5 files changed, 143 deletions(-) delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile deleted file mode 100644 index a785604018a..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM python:3.11-slim-bullseye AS build - -RUN apt-get update && \ - apt-get install --no-install-suggests --no-install-recommends --yes git - -WORKDIR /bytewax - -# Copy dataflow code -COPY sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py /bytewax -COPY sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py /bytewax - -# Copy entrypoint -COPY sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh /bytewax - -# Copy necessary parts of the Feast codebase -COPY sdk/python sdk/python -COPY protos protos -COPY go go -COPY setup.py setup.py -COPY pyproject.toml pyproject.toml -COPY README.md README.md - -# Install Feast for AWS with Bytewax dependencies -# We need this mount thingy because setuptools_scm needs access to the -# git dir to infer the version of feast we're installing. -# https://github.com/pypa/setuptools_scm#usage-from-docker -# I think it also assumes that this dockerfile is being built from the root of the directory. -RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[aws,gcp,bytewax,snowflake,postgres]' - diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py b/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py index e108c03d801..222dc380143 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py @@ -1,4 +1,3 @@ -from .bytewax_materialization_dataflow import BytewaxMaterializationDataflow from .bytewax_materialization_engine import ( BytewaxMaterializationEngine, BytewaxMaterializationEngineConfig, @@ -7,7 +6,6 @@ __all__ = [ "BytewaxMaterializationTask", - "BytewaxMaterializationDataflow", "BytewaxMaterializationEngine", "BytewaxMaterializationEngineConfig", ] diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py deleted file mode 100644 index 17cd8c3b805..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging -import os -from typing import List - -import pyarrow as pa -import pyarrow.parquet as pq -from bytewax.dataflow import Dataflow # type: ignore - -from feast import FeatureStore, FeatureView, RepoConfig -from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping - -logger = logging.getLogger(__name__) -DEFAULT_BATCH_SIZE = 1000 - - -class BytewaxMaterializationDataflow: - def __init__( - self, - config: RepoConfig, - feature_view: FeatureView, - paths: List[str], - worker_index: int, - ): - self.config = config - self.feature_store = FeatureStore(config=config) - - self.feature_view = feature_view - self.worker_index = worker_index - self.paths = paths - self.mini_batch_size = int( - os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE) - ) - - self._run_dataflow() - - def process_path(self, path): - logger.info(f"Processing path {path}") - dataset = pq.ParquetDataset(path, use_legacy_dataset=False) - batches = [] - for fragment in dataset.fragments: - for batch in fragment.to_table().to_batches( - max_chunksize=self.mini_batch_size - ): - batches.append(batch) - - return batches - - def input_builder(self, worker_index, worker_count, _state): - return [(None, self.paths[self.worker_index])] - - def output_builder(self, worker_index, worker_count): - def output_fn(mini_batch): - table: pa.Table = pa.Table.from_batches([mini_batch]) - - if self.feature_view.batch_source.field_mapping is not None: - table = _run_pyarrow_field_mapping( - table, self.feature_view.batch_source.field_mapping - ) - join_key_to_value_type = { - entity.name: entity.dtype.to_value_type() - for entity in self.feature_view.entity_columns - } - rows_to_write = _convert_arrow_to_proto( - table, self.feature_view, join_key_to_value_type - ) - self.feature_store._get_provider().online_write_batch( - config=self.config, - table=self.feature_view, - data=rows_to_write, - progress=None, - ) - - return output_fn - - def _run_dataflow(self): - flow = Dataflow() - flow.input("inp", ManualInputConfig(self.input_builder)) - flow.flat_map(self.process_path) - flow.capture(ManualOutputConfig(self.output_builder)) - cluster_main(flow, [], 0) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py deleted file mode 100644 index bbc32cc1651..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging -import os - -import yaml - -from feast import FeatureStore, RepoConfig -from feast.infra.materialization.contrib.bytewax.bytewax_materialization_dataflow import ( - BytewaxMaterializationDataflow, -) - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - with open("/var/feast/feature_store.yaml") as f: - feast_config = yaml.load(f, Loader=yaml.Loader) - - with open("/var/feast/bytewax_materialization_config.yaml") as b: - bytewax_config = yaml.load(b, Loader=yaml.Loader) - - config = RepoConfig(**feast_config) - store = FeatureStore(config=config) - - job = BytewaxMaterializationDataflow( - config, - store.get_feature_view(bytewax_config["feature_view"]), - bytewax_config["paths"], - int(os.environ["JOB_COMPLETION_INDEX"]), - ) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh b/sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh deleted file mode 100644 index 0179e5481fa..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd /bytewax -python dataflow.py From f0971a868f9a7b6e172d01a01a2b54713359e951 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Wed, 27 Mar 2024 17:18:52 -0400 Subject: [PATCH 03/20] Removing duplicate dependency ibis which was added by mistake. this should fix the failure of python unit test cases. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- .../requirements/py3.11-ci-requirements.txt | 18 +++++++------- .../requirements/py3.11-requirements.txt | 24 +++---------------- setup.py | 1 - 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 6b724b8136a..d2b58e8a196 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -41,8 +41,6 @@ attrs==23.2.0 # bowler # jsonschema # referencing -avro==1.10.0 - # via feast (setup.py) azure-core==1.30.1 # via # azure-identity @@ -63,11 +61,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.71 +boto3==1.34.72 # via # feast (setup.py) # moto -botocore==1.34.71 +botocore==1.34.72 # via # boto3 # moto @@ -288,7 +286,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httplib2==0.22.0 # via @@ -300,8 +298,6 @@ httpx==0.27.0 # via # feast (setup.py) # jupyterlab -ibis==3.3.0 - # via feast (setup.py) ibis-framework==8.0.0 # via # feast (setup.py) @@ -567,7 +563,6 @@ prompt-toolkit==3.0.43 # via ipython proto-plus==1.23.0 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -620,11 +615,11 @@ pyarrow==15.0.2 # snowflake-connector-python pyarrow-hotfix==0.6 # via ibis-framework -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 # via feast (setup.py) @@ -675,6 +670,7 @@ pytest==7.4.4 # feast (setup.py) # pytest-benchmark # pytest-cov + # pytest-env # pytest-lazy-fixture # pytest-mock # pytest-ordering @@ -684,6 +680,8 @@ pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 # via feast (setup.py) pytest-mock==1.10.4 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 90d336a40aa..fb8063327d2 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -8,7 +8,6 @@ annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via - # httpx # starlette # watchfiles appdirs==1.4.4 @@ -21,10 +20,7 @@ attrs==23.2.0 bowler==0.9.0 # via feast (setup.py) certifi==2024.2.2 - # via - # httpcore - # httpx - # requests + # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -53,21 +49,12 @@ greenlet==3.0.3 gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.4 - # via httpx + # via uvicorn httptools==0.6.1 # via uvicorn -httpx==0.27.0 - # via feast (setup.py) -ibis==3.3.0 - # via feast (setup.py) idna==3.6 # via # anyio - # httpx # requests importlib-metadata==6.11.0 # via @@ -108,13 +95,10 @@ pandas==2.2.1 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.23.0 - # via feast (setup.py) protobuf==4.25.3 # via # feast (setup.py) # mypy-protobuf - # proto-plus pyarrow==15.0.2 # via feast (setup.py) pydantic==2.6.4 @@ -149,9 +133,7 @@ rpds-py==0.18.0 six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio sqlalchemy[mypy]==1.4.52 # via # feast (setup.py) diff --git a/setup.py b/setup.py index 9d3ecf97794..23911bae1d1 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,6 @@ "bowler", # Needed for automatic repo upgrades "importlib-resources>=6.0.0,<7", "importlib_metadata>=6.8.0,<7", - "ibis~=3.3.0", ] GCP_REQUIRED = [ From 1cfa4d5ba99a19de6b6b804528a8ff2664a1f729 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Mon, 1 Apr 2024 14:57:41 -0400 Subject: [PATCH 04/20] Removing the bytewax materialization engine. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- .../contrib/bytewax/__init__.py | 11 -- .../bytewax/bytewax_materialization_engine.py | 142 ------------------ .../bytewax/bytewax_materialization_task.py | 10 -- 3 files changed, 163 deletions(-) delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py delete mode 100644 sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py b/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py deleted file mode 100644 index 222dc380143..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from .bytewax_materialization_engine import ( - BytewaxMaterializationEngine, - BytewaxMaterializationEngineConfig, -) -from .bytewax_materialization_task import BytewaxMaterializationTask - -__all__ = [ - "BytewaxMaterializationTask", - "BytewaxMaterializationEngine", - "BytewaxMaterializationEngineConfig", -] diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py deleted file mode 100644 index 05d3f87af24..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ /dev/null @@ -1,142 +0,0 @@ -import logging -import uuid -from datetime import datetime -from time import sleep -from typing import Callable, List, Literal, Sequence, Union - -import yaml -from pydantic import StrictStr -from tqdm import tqdm - -from feast import FeatureView, RepoConfig -from feast.batch_feature_view import BatchFeatureView -from feast.entity import Entity -from feast.infra.materialization.batch_materialization_engine import ( - BatchMaterializationEngine, - MaterializationJob, - MaterializationJobStatus, - MaterializationTask, -) -from feast.infra.offline_stores.offline_store import OfflineStore -from feast.infra.online_stores.online_store import OnlineStore -from feast.infra.registry.base_registry import BaseRegistry -from feast.repo_config import FeastConfigBaseModel -from feast.stream_feature_view import StreamFeatureView -from feast.utils import _get_column_names - -from ...local_engine import LocalMaterializationEngine, LocalMaterializationEngineConfig - -logger = logging.getLogger(__name__) - - -class BytewaxMaterializationEngineConfig(LocalMaterializationEngineConfig): - """Batch Materialization Engine config for Bytewax""" - - type: Literal["bytewax"] = "bytewax" - """ Materialization type selector""" - - namespace: StrictStr = "default" - """ (optional) The namespace in Kubernetes to use when creating services, configuration maps and jobs. - """ - - image: StrictStr = "bytewax/bytewax-feast:latest" - """ (optional) The container image to use when running the materialization job.""" - - env: List[dict] = [] - """ (optional) A list of environment variables to set in the created Kubernetes pods. - These environment variables can be used to reference Kubernetes secrets. - """ - - image_pull_secrets: List[dict] = [] - """ (optional) The secrets to use when pulling the image to run for the materialization job """ - - resources: dict = {} - """ (optional) The resource requests and limits for the materialization containers """ - - service_account_name: StrictStr = "" - """ (optional) The service account name to use when running the job """ - - annotations: dict = {} - """ (optional) Annotations to apply to the job container. Useful for linking the service account to IAM roles, operational metadata, etc """ - - include_security_context_capabilities: bool = True - """ (optional) Include security context capabilities in the init and job container spec """ - - labels: dict = {} - """ (optional) additional labels to append to kubernetes objects """ - - max_parallelism: int = 10 - """ (optional) Maximum number of pods allowed to run in parallel""" - - synchronous: bool = False - """ (optional) If true, wait for materialization for one feature to complete before moving to the next """ - - retry_limit: int = 2 - """ (optional) Maximum number of times to retry a materialization worker pod""" - - mini_batch_size: int = 1000 - """ (optional) Number of rows to process per write operation (default 1000)""" - - active_deadline_seconds: int = 86400 - """ (optional) Maximum amount of time a materialization job is allowed to run""" - - job_batch_size: int = 100 - """ (optional) Maximum number of pods to process per job. Only applies to synchronous materialization""" - - print_pod_logs_on_failure: bool = True - """(optional) Print pod logs on job failure. Only applies to synchronous materialization""" - - -class BytewaxMaterializationEngine(LocalMaterializationEngine): - def __init__( - self, - *, - repo_config: RepoConfig, - offline_store: OfflineStore, - online_store: OnlineStore, - **kwargs, - ): - super().__init__( - repo_config=repo_config, - offline_store=offline_store, - online_store=online_store, - **kwargs, - ) - self.repo_config = repo_config - self.offline_store = offline_store - self.online_store = online_store - - self.batch_engine_config = repo_config.batch_engine - self.namespace = self.batch_engine_config.namespace - - def update( - self, - project: str, - views_to_delete: Sequence[ - Union[BatchFeatureView, StreamFeatureView, FeatureView] - ], - views_to_keep: Sequence[ - Union[BatchFeatureView, StreamFeatureView, FeatureView] - ], - entities_to_delete: Sequence[Entity], - entities_to_keep: Sequence[Entity], - ): - """This method ensures that any necessary infrastructure or resources needed by the - engine are set up ahead of materialization.""" - pass - - def teardown_infra( - self, - project: str, - fvs: Sequence[Union[BatchFeatureView, StreamFeatureView, FeatureView]], - entities: Sequence[Entity], - ): - """This method ensures that any infrastructure or resources set up by ``update()``are torn down.""" - pass - - def materialize( - self, - registry: BaseRegistry, - tasks: List[MaterializationTask], - ) -> List[MaterializationJob]: - super().materialize(self, registry, tasks) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py deleted file mode 100644 index 8bb8da741aa..00000000000 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_task.py +++ /dev/null @@ -1,10 +0,0 @@ -from feast.infra.materialization.batch_materialization_engine import MaterializationTask - - -class BytewaxMaterializationTask(MaterializationTask): - def __init__(self, project, feature_view, start_date, end_date, tqdm): - self.project = project - self.feature_view = feature_view - self.start_date = start_date - self.end_date = end_date - self.tqdm = tqdm From 822cd6799054460890cb6b08d3162b7a90b24c18 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Mon, 1 Apr 2024 15:04:07 -0400 Subject: [PATCH 05/20] Removing the bytewax documentation. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- docs/SUMMARY.md | 1 - .../running-feast-in-production.md | 22 +---- .../batch-materialization/bytewax.md | 99 ------------------- 3 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 docs/reference/batch-materialization/bytewax.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8affea898ef..bcad64f3e13 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -106,7 +106,6 @@ * [Amazon Web Services](reference/providers/amazon-web-services.md) * [Azure](reference/providers/azure.md) * [Batch Materialization Engines](reference/batch-materialization/README.md) - * [Bytewax](reference/batch-materialization/bytewax.md) * [Snowflake](reference/batch-materialization/snowflake.md) * [AWS Lambda (alpha)](reference/batch-materialization/lambda.md) * [Spark (contrib)](reference/batch-materialization/spark.md) diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 9d1984d7366..ca33cce1031 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -57,29 +57,9 @@ To keep your online store up to date, you need to run a job that loads feature d Out of the box, Feast's materialization process uses an in-process materialization engine. This engine loads all the data being materialized into memory from the offline store, and writes it into the online store. This approach may not scale to large amounts of data, which users of Feast may be dealing with in production. -In this case, we recommend using one of the more [scalable materialization engines](./scaling-feast.md#scaling-materialization), such as the [Bytewax Materialization Engine](../reference/batch-materialization/bytewax.md), or the [Snowflake Materialization Engine](../reference/batch-materialization/snowflake.md). +In this case, we recommend using one of the more [scalable materialization engines](./scaling-feast.md#scaling-materialization), such as the [Snowflake Materialization Engine](../reference/batch-materialization/snowflake.md). Users may also need to [write a custom materialization engine](../how-to-guides/customizing-feast/creating-a-custom-materialization-engine.md) to work on their existing infrastructure. -The Bytewax materialization engine can run materialization on an existing Kubernetes cluster. An example configuration of this in a `feature_store.yaml` is as follows: - -```yaml -batch_engine: - type: bytewax - namespace: bytewax - image: bytewax/bytewax-feast:latest - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-secret-access-key -``` - ### 2.2 Scheduled materialization with Airflow > See also [data ingestion](../getting-started/concepts/data-ingestion.md#batch-data-ingestion) for code snippets diff --git a/docs/reference/batch-materialization/bytewax.md b/docs/reference/batch-materialization/bytewax.md deleted file mode 100644 index 6a97bd391db..00000000000 --- a/docs/reference/batch-materialization/bytewax.md +++ /dev/null @@ -1,99 +0,0 @@ -# Bytewax - -## Description - -The [Bytewax](https://bytewax.io) batch materialization engine provides an execution -engine for batch materializing operations (`materialize` and `materialize-incremental`). - -### Guide - -In order to use the Bytewax materialization engine, you will need a [Kubernetes](https://kubernetes.io/) cluster running version 1.22.10 or greater. - -#### Kubernetes Authentication - -The Bytewax materialization engine loads authentication and cluster information from the [kubeconfig file](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/). By default, kubectl looks for a file named `config` in the `$HOME/.kube directory`. You can specify other kubeconfig files by setting the `KUBECONFIG` environment variable. - -#### Resource Authentication - -Bytewax jobs can be configured to access [Kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/) as environment variables to access online and offline stores during job runs. - -To configure secrets, first create them using `kubectl`: - -``` shell -kubectl create secret generic -n bytewax aws-credentials --from-literal=aws-access-key-id='' --from-literal=aws-secret-access-key='' -``` - -If your Docker registry requires authentication to store/pull containers, you can use this same approach to store your repository access credential and use when running the materialization engine. - -Then configure them in the batch_engine section of `feature_store.yaml`: - -``` yaml -batch_engine: - type: bytewax - namespace: bytewax - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-secret-access-key - image_pull_secrets: - - docker-repository-access-secret -``` - -#### Configuration - -The Bytewax materialization engine is configured through the The `feature_store.yaml` configuration file: - -``` yaml -batch_engine: - type: bytewax - namespace: bytewax - image: bytewax/bytewax-feast:latest - image_pull_secrets: - - my_container_secret - service_account_name: my-k8s-service-account - include_security_context_capabilities: false - annotations: - # example annotation you might include if running on AWS EKS - iam.amazonaws.com/role: arn:aws:iam:::role/MyBytewaxPlatformRole - resources: - limits: - cpu: 1000m - memory: 2048Mi - requests: - cpu: 500m - memory: 1024Mi -``` - -**Notes:** - -* The `namespace` configuration directive specifies which Kubernetes [namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) jobs, services and configuration maps will be created in. -* The `image_pull_secrets` configuration directive specifies the pre-configured secret to use when pulling the image container from your registry. -* The `service_account_name` specifies which Kubernetes service account to run the job under. -* The `include_security_context_capabilities` flag indicates whether or not `"add": ["NET_BIND_SERVICE"]` and `"drop": ["ALL"]` are included in the job & pod security context capabilities. -* `annotations` allows you to include additional Kubernetes annotations to the job. This is particularly useful for IAM roles which grant the running pod access to cloud platform resources (for example). -* The `resources` configuration directive sets the standard Kubernetes [resource requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for the job containers to utilise when materializing data. - -#### Building a custom Bytewax Docker image - -The `image` configuration directive specifies which container image to use when running the materialization job. To create a custom image based on this container, run the following command: - -``` shell -DOCKER_BUILDKIT=1 docker build . -f ./sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile -t -``` - -Once that image is built and pushed to a registry, it can be specified as a part of the batch engine configuration: - -``` shell -batch_engine: - type: bytewax - namespace: bytewax - image: -``` - From 10ddd014198367ac7c72ca9d6977fe4bf375f606 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Tue, 2 Apr 2024 11:29:51 -0400 Subject: [PATCH 06/20] Adding missed duckdb dependencies. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- sdk/python/requirements/py3.11-ci-requirements.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index d2b58e8a196..eeef03af0f5 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -158,6 +158,12 @@ docker==7.0.0 # testcontainers docutils==0.19 # via sphinx +duckdb==0.10.1 + # via + # duckdb-engine + # ibis-framework +duckdb-engine==0.11.2 + # via ibis-framework entrypoints==0.4 # via altair execnet==2.0.2 @@ -298,7 +304,7 @@ httpx==0.27.0 # via # feast (setup.py) # jupyterlab -ibis-framework==8.0.0 +ibis-framework[duckdb]==8.0.0 # via # feast (setup.py) # ibis-substrait @@ -823,8 +829,13 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.52 # via + # duckdb-engine # feast (setup.py) + # ibis-framework # sqlalchemy + # sqlalchemy-views +sqlalchemy-views==0.3.2 + # via ibis-framework sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy sqlglot==20.11.0 From b8fdfbd8f19ab4a886d6f1e80bb95ce33e1a474f Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 12:28:29 -0400 Subject: [PATCH 07/20] updating the requirements.txt files as per the latest code. Signed-off-by: Lokesh Rangineni --- .../requirements/py3.11-ci-requirements.txt | 66 ++++++++++--------- .../requirements/py3.11-requirements.txt | 15 ++--- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index eeef03af0f5..5a84999d888 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -57,8 +57,6 @@ beautifulsoup4==4.12.3 # via nbconvert bidict==0.23.1 # via ibis-framework -black==22.12.0 - # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.34.72 @@ -86,6 +84,7 @@ certifi==2024.2.2 # via # httpcore # httpx + # kubernetes # minio # requests # snowflake-connector-python @@ -102,7 +101,6 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # black # bowler # dask # feast (setup.py) @@ -136,7 +134,7 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.1 +dask[array,dataframe]==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -146,8 +144,8 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deprecation==2.1.0 - # via testcontainers +deltalake==0.17.2 + # via feast (setup.py) dill==0.3.8 # via feast (setup.py) distlib==0.3.8 @@ -182,8 +180,6 @@ firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler -flake8==6.0.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 @@ -215,6 +211,7 @@ google-auth==2.29.0 # google-auth-httplib2 # google-cloud-core # google-cloud-storage + # kubernetes google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 @@ -280,7 +277,7 @@ grpcio-testing==1.62.1 # via feast (setup.py) grpcio-tools==1.62.1 # via feast (setup.py) -gunicorn==21.2.0 +gunicorn==21.2.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via @@ -322,11 +319,7 @@ idna==3.6 imagesize==1.4.1 # via sphinx importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.4.0 - # via feast (setup.py) + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.3 @@ -342,8 +335,6 @@ isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.13.2 - # via feast (setup.py) jedi==0.19.1 # via ipython jinja2==3.1.3 @@ -416,6 +407,8 @@ jupyterlab-server==2.25.4 # notebook jupyterlab-widgets==3.0.10 # via ipywidgets +kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -433,8 +426,6 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -mccabe==0.7.0 - # via flake8 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 @@ -466,9 +457,7 @@ mypy==1.9.0 # feast (setup.py) # sqlalchemy mypy-extensions==1.0.0 - # via - # black - # mypy + # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) nbclient==0.10.0 @@ -494,6 +483,7 @@ notebook-shim==0.2.4 numpy==1.24.4 # via # altair + # dask # db-dtypes # feast (setup.py) # great-expectations @@ -501,6 +491,8 @@ numpy==1.24.4 # pandas # pyarrow # scipy +oauthlib==3.2.2 + # via requests-oauthlib overrides==7.7.0 # via jupyter-server packaging==24.0 @@ -508,7 +500,6 @@ packaging==24.0 # build # dask # db-dtypes - # deprecation # docker # google-cloud-bigquery # great-expectations @@ -527,6 +518,7 @@ packaging==24.0 pandas==2.2.1 # via # altair + # dask # db-dtypes # feast (setup.py) # google-cloud-bigquery @@ -541,8 +533,6 @@ parsy==2.1 # via ibis-framework partd==1.4.1 # via dask -pathspec==0.12.1 - # via black pbr==6.0.0 # via mock pexpect==4.9.0 @@ -551,7 +541,6 @@ pip-tools==7.4.1 # via feast (setup.py) platformdirs==3.11.0 # via - # black # jupyter-core # snowflake-connector-python # virtualenv @@ -615,12 +604,15 @@ py4j==0.10.9.7 pyarrow==15.0.2 # via # db-dtypes + # deltalake # feast (setup.py) # google-cloud-bigquery # ibis-framework # snowflake-connector-python pyarrow-hotfix==0.6 - # via ibis-framework + # via + # deltalake + # ibis-framework pyasn1==0.6.0 # via # pyasn1-modules @@ -629,8 +621,6 @@ pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 # via feast (setup.py) -pycodestyle==2.10.0 - # via flake8 pycparser==2.21 # via cffi pydantic==2.6.4 @@ -640,8 +630,6 @@ pydantic==2.6.4 # great-expectations pydantic-core==2.16.3 # via pydantic -pyflakes==3.0.1 - # via flake8 pygments==2.17.2 # via # feast (setup.py) @@ -706,6 +694,7 @@ python-dateutil==2.9.0.post0 # great-expectations # ibis-framework # jupyter-client + # kubernetes # moto # pandas # rockset @@ -727,6 +716,7 @@ pyyaml==6.0.1 # feast (setup.py) # ibis-substrait # jupyter-events + # kubernetes # pre-commit # responses # uvicorn @@ -755,12 +745,16 @@ requests==2.31.0 # google-cloud-storage # great-expectations # jupyterlab-server + # kubernetes # moto # msal + # requests-oauthlib # responses # snowflake-connector-python # sphinx # trino +requests-oauthlib==2.0.0 + # via kubernetes responses==0.25.0 # via moto rfc3339-validator==0.1.4 @@ -783,6 +777,8 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations +ruff==0.4.1 + # via feast (setup.py) s3transfer==0.10.1 # via boto3 scipy==1.12.0 @@ -797,6 +793,7 @@ six==1.16.0 # geomet # happybase # isodate + # kubernetes # mock # python-dateutil # rfc3339-validator @@ -854,7 +851,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==3.7.1 +testcontainers==4.3.3 # via feast (setup.py) thriftpy2==0.4.20 # via happybase @@ -939,6 +936,7 @@ typing-extensions==4.10.0 # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs + # testcontainers # typeguard tzdata==2024.1 # via pandas @@ -956,10 +954,12 @@ urllib3==1.26.18 # docker # feast (setup.py) # great-expectations + # kubernetes # minio # requests # responses # rockset + # testcontainers uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 @@ -981,7 +981,9 @@ webencodings==0.5.1 # bleach # tinycss2 websocket-client==1.7.0 - # via jupyter-server + # via + # jupyter-server + # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index fb8063327d2..7a7813511ae 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -34,7 +34,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.1 +dask[array,dataframe]==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -46,7 +46,7 @@ fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 +gunicorn==21.2.0 ; platform_system != "Windows" # via feast (setup.py) h11==0.14.0 # via uvicorn @@ -57,11 +57,7 @@ idna==3.6 # anyio # requests importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.4.0 - # via feast (setup.py) + # via dask jinja2==3.1.3 # via feast (setup.py) jsonschema==4.21.1 @@ -84,6 +80,7 @@ mypy-protobuf==3.5.0 # via feast (setup.py) numpy==1.24.4 # via + # dask # feast (setup.py) # pandas # pyarrow @@ -92,7 +89,9 @@ packaging==24.0 # dask # gunicorn pandas==2.2.1 - # via feast (setup.py) + # via + # dask + # feast (setup.py) partd==1.4.1 # via dask protobuf==4.25.3 From 4bec6b0fe3589e1dc60aa8533a3e4cdd278cc8dc Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 12:28:29 -0400 Subject: [PATCH 08/20] updating the requirements.txt files as per the latest code. Signed-off-by: Lokesh Rangineni --- .../requirements/py3.10-ci-requirements.txt | 30 +++--------- .../requirements/py3.10-requirements.txt | 14 ++---- .../requirements/py3.9-ci-requirements.txt | 46 ++++--------------- .../requirements/py3.9-requirements.txt | 19 +++----- 4 files changed, 26 insertions(+), 83 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 15dadfd8740..55c23e151a5 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -32,8 +34,6 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,11 +168,6 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -837,7 +832,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # duckdb-engine # feast (setup.py) @@ -846,6 +841,8 @@ sqlalchemy[mypy]==2.0.29 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -870,16 +867,6 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pyproject-hooks - # pytest - # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -951,8 +938,6 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi @@ -963,10 +948,9 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy + # sqlalchemy2-stubs # testcontainers # typeguard - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 2cfe62c55bd..b75cfe18553 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt @@ -42,8 +42,6 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.2.1 - # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -142,10 +140,12 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # feast (setup.py) # sqlalchemy +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -154,8 +154,6 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -168,14 +166,12 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy + # sqlalchemy2-stubs # typeguard - # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 8de9151e4e3..97b85d3e543 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -32,8 +34,6 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,11 +168,6 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -329,16 +324,7 @@ idna==3.7 imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 - # via - # build - # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx - # typeguard + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.4 @@ -799,8 +785,6 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml ruff==0.4.1 # via feast (setup.py) s3transfer==0.10.1 @@ -848,7 +832,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # duckdb-engine # feast (setup.py) @@ -857,6 +841,8 @@ sqlalchemy[mypy]==2.0.29 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -881,16 +867,6 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pyproject-hooks - # pytest - # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -962,23 +938,18 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi # great-expectations # ibis-framework - # ipython # mypy # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy - # starlette + # sqlalchemy2-stubs # testcontainers # typeguard - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 @@ -1000,7 +971,6 @@ urllib3==1.26.18 # requests # responses # rockset - # snowflake-connector-python # testcontainers uvicorn[standard]==0.29.0 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 472f3e90b99..56e1aef446a 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt @@ -42,8 +42,6 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.2.1 - # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -63,9 +61,7 @@ idna==3.7 # anyio # requests importlib-metadata==7.1.0 - # via - # dask - # typeguard + # via dask jinja2==3.1.3 # via feast (setup.py) jsonschema==4.21.1 @@ -144,10 +140,12 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # feast (setup.py) # sqlalchemy +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -156,8 +154,6 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -170,15 +166,12 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy - # starlette + # sqlalchemy2-stubs # typeguard - # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 From fb6d0e5cb88d90ec4fea86c86b3bd5d195e7f938 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 12:41:01 -0400 Subject: [PATCH 09/20] Revert "updating the requirements.txt files as per the latest code." Signed-off-by: Lokesh Rangineni This reverts commit 4bec6b0fe3589e1dc60aa8533a3e4cdd278cc8dc. --- .../requirements/py3.10-ci-requirements.txt | 30 +++++++++--- .../requirements/py3.10-requirements.txt | 14 ++++-- .../requirements/py3.9-ci-requirements.txt | 46 +++++++++++++++---- .../requirements/py3.9-requirements.txt | 19 +++++--- 4 files changed, 83 insertions(+), 26 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 55c23e151a5..15dadfd8740 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,8 +18,6 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -34,6 +32,8 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab +async-timeout==4.0.3 + # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,6 +168,11 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair +exceptiongroup==1.2.1 + # via + # anyio + # ipython + # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -832,7 +837,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # duckdb-engine # feast (setup.py) @@ -841,8 +846,6 @@ sqlalchemy[mypy]==1.4.52 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -867,6 +870,16 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pyproject-hooks + # pytest + # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -938,6 +951,8 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # anyio + # async-lru # azure-core # azure-storage-blob # fastapi @@ -948,9 +963,10 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy2-stubs + # sqlalchemy # testcontainers # typeguard + # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index b75cfe18553..2cfe62c55bd 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt @@ -42,6 +42,8 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) +exceptiongroup==1.2.1 + # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -140,12 +142,10 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -154,6 +154,8 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via mypy toolz==0.12.1 # via # dask @@ -166,12 +168,14 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via + # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy2-stubs + # sqlalchemy # typeguard + # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 97b85d3e543..8de9151e4e3 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt @@ -18,8 +18,6 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -34,6 +32,8 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab +async-timeout==4.0.3 + # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,6 +168,11 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair +exceptiongroup==1.2.1 + # via + # anyio + # ipython + # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -324,7 +329,16 @@ idna==3.7 imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 - # via dask + # via + # build + # dask + # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # nbconvert + # sphinx + # typeguard iniconfig==2.0.0 # via pytest ipykernel==6.29.4 @@ -785,6 +799,8 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml ruff==0.4.1 # via feast (setup.py) s3transfer==0.10.1 @@ -832,7 +848,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # duckdb-engine # feast (setup.py) @@ -841,8 +857,6 @@ sqlalchemy[mypy]==1.4.52 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -867,6 +881,16 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pyproject-hooks + # pytest + # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -938,18 +962,23 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # anyio + # async-lru # azure-core # azure-storage-blob # fastapi # great-expectations # ibis-framework + # ipython # mypy # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy2-stubs + # sqlalchemy + # starlette # testcontainers # typeguard + # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 @@ -971,6 +1000,7 @@ urllib3==1.26.18 # requests # responses # rockset + # snowflake-connector-python # testcontainers uvicorn[standard]==0.29.0 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 56e1aef446a..472f3e90b99 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt @@ -42,6 +42,8 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) +exceptiongroup==1.2.1 + # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -61,7 +63,9 @@ idna==3.7 # anyio # requests importlib-metadata==7.1.0 - # via dask + # via + # dask + # typeguard jinja2==3.1.3 # via feast (setup.py) jsonschema==4.21.1 @@ -140,12 +144,10 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -154,6 +156,8 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via mypy toolz==0.12.1 # via # dask @@ -166,12 +170,15 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via + # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy2-stubs + # sqlalchemy + # starlette # typeguard + # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 From d089b0bcfb2013d0c6933ae1413d528d3ffd9174 Mon Sep 17 00:00:00 2001 From: lokeshrangineni Date: Thu, 25 Apr 2024 12:26:18 -0400 Subject: [PATCH 10/20] fix: Changed the code the way mysql container is initialized. (#4140) * chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * chore: Incorporate release 0.37.1 to master (#4127) * chore(release): release 0.37.1 ## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) ### Bug Fixes * Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([1a1f0b1](https://github.com/feast-dev/feast/commit/1a1f0b1c56aa2ac00b1e1aa1e21cc200ea659334)) ### Reverts * Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([d5ded69](https://github.com/feast-dev/feast/commit/d5ded69dea9af3a363feaa948cd3d2dcf10fb80c)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) * chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko --------- Signed-off-by: tokoko Co-authored-by: feast-ci-bot Co-authored-by: Tornike Gurgenidze Signed-off-by: Lokesh Rangineni * chore: Install python dependencies with uv in workflows (#4086) * install dependencies in unit-tests with uv Signed-off-by: tokoko * install dependencies in unit-tests with uv Signed-off-by: tokoko * enable caching, change linter job Signed-off-by: tokoko * change local integration tests to uv Signed-off-by: tokoko * change all installs to uv Signed-off-by: tokoko * try adding uv cache Signed-off-by: tokoko * fix lambda cache step name Signed-off-by: tokoko * reenable caches for uv Signed-off-by: tokoko * remove dangling cache step Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * feat: Make arrow primary interchange for online ODFV execution (#4143) * rewrite online flow to use transform_arrow Signed-off-by: tokoko * fix transformation server Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * feat: Feast/IKV online store documentation (#4146) * feat: Feast/IKV online store documentation Signed-off-by: Pushkar Gupta * functionality matric Signed-off-by: Pushkar Gupta * more changes Signed-off-by: Pushkar Gupta * mount dir Signed-off-by: Pushkar Gupta --------- Signed-off-by: Pushkar Gupta Signed-off-by: Lokesh Rangineni * fix: Default value is not set in Redis connection string using environment variable (#4136) Removed documentation of Redis connection string supporting default values when using environment variables as it isn't supported Fixes #3669 Signed-off-by: Theodor Mihalache Signed-off-by: Lokesh Rangineni * fix: Make sure schema is used when calling `get_table_query_string` method for Snowflake datasource (#4131) * Fix get_table_query_string method for Snowflake datasource Signed-off-by: TomSteenbergen * Add quotes to table string Signed-off-by: TomSteenbergen --------- Signed-off-by: TomSteenbergen Signed-off-by: Lokesh Rangineni * fix: Change checkout action back to v3 from v5 which isn't released yet (#4147) Signed-off-by: Jeremy Ary Signed-off-by: Lokesh Rangineni * changed the code the way mysql container is initialized. Trying to fix the issue - https://github.com/feast-dev/feast/issues/4128 Also going to check if this change will be resolved in the github actions as well. Signed-off-by: Lokesh Rangineni * reformatted the file to resolve lint error. Signed-off-by: Lokesh Rangineni * reformatted the file to resolve lint error. Signed-off-by: Lokesh Rangineni --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni Signed-off-by: Pushkar Gupta Signed-off-by: Theodor Mihalache Signed-off-by: TomSteenbergen Signed-off-by: Jeremy Ary Co-authored-by: Tornike Gurgenidze Co-authored-by: Jeremy Ary Co-authored-by: feast-ci-bot Co-authored-by: Pushkar Gupta Co-authored-by: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Co-authored-by: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> --- sdk/python/tests/unit/test_sql_registry.py | 30 +++++++--------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 094b8967c18..a1460663aeb 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -21,6 +21,7 @@ from pytest_lazyfixture import lazy_fixture from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.mysql import MySqlContainer from feast import FileSource, RequestSource from feast.data_format import ParquetFormat @@ -41,7 +42,6 @@ POSTGRES_PASSWORD = "test" POSTGRES_DB = "test" - logger = logging.getLogger(__name__) @@ -81,32 +81,20 @@ def pg_registry(): @pytest.fixture(scope="session") def mysql_registry(): - container = ( - DockerContainer("mysql:latest") - .with_exposed_ports(3306) - .with_env("MYSQL_RANDOM_ROOT_PASSWORD", "true") - .with_env("MYSQL_USER", POSTGRES_USER) - .with_env("MYSQL_PASSWORD", POSTGRES_PASSWORD) - .with_env("MYSQL_DATABASE", POSTGRES_DB) - ) - + container = MySqlContainer("mysql:latest") container.start() - # The log string uses '8.0.*' since the version might be changed as new Docker images are pushed. - log_string_to_wait_for = "/usr/sbin/mysqld: ready for connections. Version: '(\\d+(\\.\\d+){1,2})' socket: '/var/run/mysqld/mysqld.sock' port: 3306" # noqa: W605 - waited = wait_for_logs( - container=container, - predicate=log_string_to_wait_for, - timeout=60, - interval=10, + # testing for the database to exist and ready to connect and start testing. + import sqlalchemy + + engine = sqlalchemy.create_engine( + container.get_connection_url(), pool_pre_ping=True ) - logger.info("Waited for %s seconds until mysql container was up", waited) - container_port = container.get_exposed_port(3306) - container_host = container.get_container_host_ip() + engine.connect() registry_config = RegistryConfig( registry_type="sql", - path=f"mysql+pymysql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + path=container.get_connection_url(), sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) From d9d59af918dbb49b961ff4e49ad3790187828646 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 14:12:57 -0400 Subject: [PATCH 11/20] just checking if this resolves the issue with dask incompatibility. Signed-off-by: Lokesh Rangineni --- sdk/python/requirements/py3.11-ci-requirements.txt | 10 ++++++++-- sdk/python/requirements/py3.11-requirements.txt | 13 ++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 5a84999d888..9e384d73ec9 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -134,8 +134,12 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask[array,dataframe]==2024.2.1 - # via feast (setup.py) +dask[array,dataframe]==2024.4.2 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.13 + # via dask db-dtypes==1.2.0 # via google-cloud-bigquery debugpy==1.8.1 @@ -519,6 +523,7 @@ pandas==2.2.1 # via # altair # dask + # dask-expr # db-dtypes # feast (setup.py) # google-cloud-bigquery @@ -603,6 +608,7 @@ py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # dask-expr # db-dtypes # deltalake # feast (setup.py) diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 7a7813511ae..d661752acbb 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -34,8 +34,12 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[array,dataframe]==2024.2.1 - # via feast (setup.py) +dask[array,dataframe]==2024.4.2 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.13 + # via dask dill==0.3.8 # via feast (setup.py) fastapi==0.110.0 @@ -91,6 +95,7 @@ packaging==24.0 pandas==2.2.1 # via # dask + # dask-expr # feast (setup.py) partd==1.4.1 # via dask @@ -99,7 +104,9 @@ protobuf==4.25.3 # feast (setup.py) # mypy-protobuf pyarrow==15.0.2 - # via feast (setup.py) + # via + # dask-expr + # feast (setup.py) pydantic==2.6.4 # via # fastapi From 89ba1d86da5832ae1001b904f115062739ff4ae9 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 14:33:06 -0400 Subject: [PATCH 12/20] updating 3.9 dependencies to make it work with 3.11 as well. Signed-off-by: Lokesh Rangineni --- sdk/python/requirements/py3.9-ci-requirements.txt | 8 ++++++-- sdk/python/requirements/py3.9-requirements.txt | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 8de9151e4e3..6d9f48520c9 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -848,7 +850,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # duckdb-engine # feast (setup.py) @@ -857,6 +859,8 @@ sqlalchemy[mypy]==2.0.29 # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -974,7 +978,7 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy + # sqlalchemy2-stubs # starlette # testcontainers # typeguard diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 472f3e90b99..0778648dd90 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -144,10 +144,12 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # feast (setup.py) # sqlalchemy +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -175,7 +177,7 @@ typing-extensions==4.11.0 # mypy # pydantic # pydantic-core - # sqlalchemy + # sqlalchemy2-stubs # starlette # typeguard # uvicorn From 2bfedcf6a9930908c675c7eecd4da2dd8f6fe48f Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 15:08:48 -0400 Subject: [PATCH 13/20] updating 3.10 dependencies to make it work with 3.11 as well. Signed-off-by: Lokesh Rangineni --- .../requirements/py3.10-ci-requirements.txt | 31 +++++-------------- .../requirements/py3.10-requirements.txt | 18 ++++------- setup.py | 2 +- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 15dadfd8740..4a1e5eeaf44 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -32,8 +34,6 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,11 +168,6 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -837,15 +832,16 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # duckdb-engine # feast (setup.py) # ibis-framework - # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -870,16 +866,6 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pyproject-hooks - # pytest - # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -951,8 +937,6 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi @@ -963,10 +947,9 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy + # sqlalchemy2-stubs # testcontainers # typeguard - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 2cfe62c55bd..970bb42c091 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt @@ -42,8 +42,6 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.2.1 - # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -142,10 +140,10 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.29 - # via - # feast (setup.py) - # sqlalchemy +sqlalchemy[mypy]==1.4.52 + # via feast (setup.py) +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -154,8 +152,6 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -168,14 +164,12 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy + # sqlalchemy2-stubs # typeguard - # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 diff --git a/setup.py b/setup.py index 9accd9866ff..94329591c15 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2021.1.0", + "dask[dataframe]>=2024.4.2", "bowler", # Needed for automatic repo upgrades ] From 29585f2ba8c2d6c7f7085bcc1ff949fd2cff30bc Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 16:02:58 -0400 Subject: [PATCH 14/20] Revert "updating 3.10 dependencies to make it work with 3.11 as well." Signed-off-by: Lokesh Rangineni This reverts commit 8d771353e684ffdb00a7cf9fc22b7bee91e4381f. --- .../requirements/py3.10-ci-requirements.txt | 31 ++++++++++++++----- .../requirements/py3.10-requirements.txt | 18 +++++++---- setup.py | 2 +- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 4a1e5eeaf44..15dadfd8740 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,8 +18,6 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -34,6 +32,8 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab +async-timeout==4.0.3 + # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,6 +168,11 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair +exceptiongroup==1.2.1 + # via + # anyio + # ipython + # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -832,16 +837,15 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # duckdb-engine # feast (setup.py) # ibis-framework + # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -866,6 +870,16 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pyproject-hooks + # pytest + # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -937,6 +951,8 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # anyio + # async-lru # azure-core # azure-storage-blob # fastapi @@ -947,9 +963,10 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy2-stubs + # sqlalchemy # testcontainers # typeguard + # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 970bb42c091..2cfe62c55bd 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt @@ -42,6 +42,8 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) +exceptiongroup==1.2.1 + # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -140,10 +142,10 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==1.4.52 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy +sqlalchemy[mypy]==2.0.29 + # via + # feast (setup.py) + # sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -152,6 +154,8 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via mypy toolz==0.12.1 # via # dask @@ -164,12 +168,14 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via + # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy2-stubs + # sqlalchemy # typeguard + # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 diff --git a/setup.py b/setup.py index 94329591c15..9accd9866ff 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2024.4.2", + "dask[dataframe]>=2021.1.0", "bowler", # Needed for automatic repo upgrades ] From bada729398ce87f3160a6a1712d940a25a84ca1b Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 16:12:03 -0400 Subject: [PATCH 15/20] updating dask version so that it can work on python 3.11 Signed-off-by: Lokesh Rangineni --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9accd9866ff..94329591c15 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2021.1.0", + "dask[dataframe]>=2024.4.2", "bowler", # Needed for automatic repo upgrades ] From 95355c40940dac8f342f5fb4120e35b23a09745d Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Thu, 25 Apr 2024 19:13:32 -0700 Subject: [PATCH 16/20] fix: Update doc (#4153) update doc Signed-off-by: cmuhao --- docs/reference/online-stores/postgres.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 277494868c8..34d4de34883 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -64,7 +64,7 @@ Below is a matrix indicating which functionality is supported by the Postgres on To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). ## PGVector -The Postgres online store supports the use of [PGVector](https://pgvector.dev/) for storing feature values. +The Postgres online store supports the use of [PGVector](https://github.com/pgvector/pgvector) for storing feature values. To enable PGVector, set `pgvector_enabled: true` in the online store configuration. The `vector_len` parameter can be used to specify the length of the vector. The default value is 512. From e7dddd336bcf9af74a7c5aa0703b2f6a41f9cffa Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Wed, 27 Mar 2024 00:56:08 -0400 Subject: [PATCH 17/20] Upgrading the python version to 3.11 and also removing the bytewax dependencies. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni --- .github/workflows/unit_tests.yml | 2 +- sdk/python/feast/infra/materialization/kubernetes/Dockerfile | 2 +- setup.py | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3276101dcb8..8dba6acafe4 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10", "3.11"] - os: [ ubuntu-latest, macOS-12 ] + os: [ ubuntu-latest, macos-12 ] exclude: - os: macos-12 python-version: "3.9" diff --git a/sdk/python/feast/infra/materialization/kubernetes/Dockerfile b/sdk/python/feast/infra/materialization/kubernetes/Dockerfile index 956287a1d6a..510bb722851 100644 --- a/sdk/python/feast/infra/materialization/kubernetes/Dockerfile +++ b/sdk/python/feast/infra/materialization/kubernetes/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-bullseye AS build +FROM python:3.11-slim-bullseye AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes git diff --git a/setup.py b/setup.py index 94329591c15..4f089a13cb8 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,10 @@ "gunicorn; platform_system != 'Windows'", "dask[dataframe]>=2024.4.2", "bowler", # Needed for automatic repo upgrades + # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). + "httpx>=0.23.3", + "importlib-resources>=6.0.0,<7", + "importlib_metadata>=6.8.0,<7", ] GCP_REQUIRED = [ @@ -164,6 +168,7 @@ "mock==2.0.0", "moto<5", "mypy>=1.4.1", + "avro==1.10.0", "urllib3>=1.25.4,<3", "psutil==5.9.0", "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 From 5d4c322262b46b9c6277aaec93dbd3ef0d35c35c Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 15:08:48 -0400 Subject: [PATCH 18/20] updating 3.10 dependencies to make it work with 3.11 as well. Signed-off-by: Lokesh Rangineni --- .../requirements/py3.10-ci-requirements.txt | 31 +++++-------------- .../requirements/py3.10-requirements.txt | 18 ++++------- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 15dadfd8740..4a1e5eeaf44 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -32,8 +34,6 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,11 +168,6 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -837,15 +832,16 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==1.4.52 # via # duckdb-engine # feast (setup.py) # ibis-framework - # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -870,16 +866,6 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pyproject-hooks - # pytest - # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -951,8 +937,6 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi @@ -963,10 +947,9 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy + # sqlalchemy2-stubs # testcontainers # typeguard - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 2cfe62c55bd..970bb42c091 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt @@ -42,8 +42,6 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.2.1 - # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -142,10 +140,10 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.29 - # via - # feast (setup.py) - # sqlalchemy +sqlalchemy[mypy]==1.4.52 + # via feast (setup.py) +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -154,8 +152,6 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -168,14 +164,12 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy + # sqlalchemy2-stubs # typeguard - # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 From e7b9c13d4937ede54392857f9cbbbbcb67439c7f Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 16:02:58 -0400 Subject: [PATCH 19/20] Revert "updating 3.10 dependencies to make it work with 3.11 as well." Signed-off-by: Lokesh Rangineni This reverts commit 8d771353e684ffdb00a7cf9fc22b7bee91e4381f. --- .../requirements/py3.10-ci-requirements.txt | 31 ++++++++++++++----- .../requirements/py3.10-requirements.txt | 18 +++++++---- setup.py | 2 +- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 4a1e5eeaf44..15dadfd8740 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,8 +18,6 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -34,6 +32,8 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab +async-timeout==4.0.3 + # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -168,6 +168,11 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair +exceptiongroup==1.2.1 + # via + # anyio + # ipython + # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -832,16 +837,15 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.52 +sqlalchemy[mypy]==2.0.29 # via # duckdb-engine # feast (setup.py) # ibis-framework + # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy sqlglot==20.11.0 # via ibis-framework stack-data==0.6.3 @@ -866,6 +870,16 @@ tinycss2==1.2.1 # via nbconvert toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pyproject-hooks + # pytest + # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -937,6 +951,8 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # anyio + # async-lru # azure-core # azure-storage-blob # fastapi @@ -947,9 +963,10 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # snowflake-connector-python - # sqlalchemy2-stubs + # sqlalchemy # testcontainers # typeguard + # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 970bb42c091..2cfe62c55bd 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt @@ -42,6 +42,8 @@ dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) +exceptiongroup==1.2.1 + # via anyio fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 @@ -140,10 +142,10 @@ six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==1.4.52 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy +sqlalchemy[mypy]==2.0.29 + # via + # feast (setup.py) + # sqlalchemy starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -152,6 +154,8 @@ tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) +tomli==2.0.1 + # via mypy toolz==0.12.1 # via # dask @@ -164,12 +168,14 @@ types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via + # anyio # fastapi # mypy # pydantic # pydantic-core - # sqlalchemy2-stubs + # sqlalchemy # typeguard + # uvicorn tzdata==2024.1 # via pandas urllib3==2.2.1 diff --git a/setup.py b/setup.py index 4f089a13cb8..ea94f3a4f59 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2024.4.2", + "dask[dataframe]>=2021.1.0", "bowler", # Needed for automatic repo upgrades # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). "httpx>=0.23.3", From 3d005b019652d469d62c77ae1668c168f0360153 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Thu, 25 Apr 2024 16:12:03 -0400 Subject: [PATCH 20/20] updating dask version so that it can work on python 3.11 Signed-off-by: Lokesh Rangineni --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ea94f3a4f59..4f089a13cb8 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2021.1.0", + "dask[dataframe]>=2024.4.2", "bowler", # Needed for automatic repo upgrades # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). "httpx>=0.23.3",