From 76432ea915bd8488fda6a9ffaab4026cfd2589a2 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Thu, 2 May 2024 11:07:46 -0500 Subject: [PATCH 1/2] chore: Bumping CI to test new cloud creds, do not merge Signed-off-by: Jeremy Ary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1e06774dac..e9b7ff47436 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

-
+
[![unit-tests](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml) [![integration-tests-and-build](https://github.com/feast-dev/feast/actions/workflows/master_only.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/master_only.yml) From ae7759d1aee7d9ea5e93aa900fa063f4b58f7f67 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Fri, 3 May 2024 10:40:06 -0500 Subject: [PATCH 2/2] chore: Update S3 CI bucket to avoid global namespace issues Signed-off-by: Jeremy Ary --- Makefile | 2 +- docs/reference/offline-stores/redshift.md | 4 ++-- .../contrib/athena_offline_store/tests/data_source.py | 2 +- .../tests/integration/feature_repos/repo_configuration.py | 2 +- .../feature_repos/universal/data_sources/redshift.py | 7 ++++--- .../tests/integration/registration/test_feature_store.py | 2 +- sdk/python/tests/integration/registration/test_registry.py | 2 +- .../tests/unit/infra/offline_stores/test_offline_store.py | 6 +++--- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index f0abc930069..3a1d0ebc808 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ test-python-universal-athena: ATHENA_DATA_SOURCE=AwsDataCatalog \ ATHENA_DATABASE=default \ ATHENA_WORKGROUP=primary \ - ATHENA_S3_BUCKET_NAME=feast-integration-tests \ + ATHENA_S3_BUCKET_NAME=feast-int-bucket \ python -m pytest -n 8 --integration \ -k "not test_go_feature_server and \ not test_logged_features_validation and \ diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md index e9bcbfeff1a..e33a1856cb2 100644 --- a/docs/reference/offline-stores/redshift.md +++ b/docs/reference/offline-stores/redshift.md @@ -130,8 +130,8 @@ The following inline policy can be used to grant Redshift necessary permissions "Action": "s3:*", "Effect": "Allow", "Resource": [ - "arn:aws:s3:::feast-integration-tests", - "arn:aws:s3:::feast-integration-tests/*" + "arn:aws:s3:::feast-int-bucket", + "arn:aws:s3:::feast-int-bucket/*" ] } ], diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index f01144afcc4..f95a750fd14 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -31,7 +31,7 @@ def __init__(self, project_name: str, *args, **kwargs): data_source = os.getenv("ATHENA_DATA_SOURCE", "AwsDataCatalog") database = os.getenv("ATHENA_DATABASE", "default") workgroup = os.getenv("ATHENA_WORKGROUP", "primary") - bucket_name = os.getenv("ATHENA_S3_BUCKET_NAME", "feast-integration-tests") + bucket_name = os.getenv("ATHENA_S3_BUCKET_NAME", "feast-int-bucket") self.client = aws_utils.get_athena_data_client(region) self.s3 = aws_utils.get_s3_resource(region) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 096744f5472..4007106a064 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -461,7 +461,7 @@ def construct_test_environment( test_repo_config.python_feature_server and test_repo_config.provider == "aws" ) or test_repo_config.registry_location == RegistryLocation.S3: aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) registry: Union[str, RegistryConfig] = ( f"{aws_registry_path}/{project}/registry.db" diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index 60fb8950a96..8fe933fbba7 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -30,16 +30,17 @@ def __init__(self, project_name: str, *args, **kwargs): self.s3 = aws_utils.get_s3_resource(os.getenv("AWS_REGION", "us-west-2")) self.offline_store_config = RedshiftOfflineStoreConfig( - cluster_id=os.getenv("AWS_CLUSTER_ID", "feast-integration-tests"), + cluster_id=os.getenv("AWS_CLUSTER_ID", "feast-int-bucket"), region=os.getenv("AWS_REGION", "us-west-2"), user=os.getenv("AWS_USER", "admin"), database=os.getenv("AWS_DB", "feast"), s3_staging_location=os.getenv( "AWS_STAGING_LOCATION", - "s3://feast-integration-tests/redshift/tests/ingestion", + "s3://feast-int-bucket/redshift/tests/ingestion", ), iam_role=os.getenv( - "AWS_IAM_ROLE", "arn:aws:iam::402087665549:role/redshift_s3_access_role" + "AWS_IAM_ROLE", + "arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631", ), workgroup="", ) diff --git a/sdk/python/tests/integration/registration/test_feature_store.py b/sdk/python/tests/integration/registration/test_feature_store.py index deb1b0635f3..bf0c2fb61fd 100644 --- a/sdk/python/tests/integration/registration/test_feature_store.py +++ b/sdk/python/tests/integration/registration/test_feature_store.py @@ -226,7 +226,7 @@ def feature_store_with_gcs_registry(): @pytest.fixture def feature_store_with_s3_registry(): aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) return FeatureStore( config=RepoConfig( diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 232f0356093..70d118ecf95 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -53,7 +53,7 @@ def gcs_registry() -> Registry: @pytest.fixture def s3_registry() -> Registry: aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) registry_config = RegistryConfig( path=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db", diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index 0232a8d379d..e5768a81b21 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -112,12 +112,12 @@ def retrieval_job(request, environment): return FileRetrievalJob(lambda: 1, full_feature_names=False) elif request.param is RedshiftRetrievalJob: offline_store_config = RedshiftOfflineStoreConfig( - cluster_id="feast-integration-tests", + cluster_id="feast-int-bucket", region="us-west-2", user="admin", database="feast", - s3_staging_location="s3://feast-integration-tests/redshift/tests/ingestion", - iam_role="arn:aws:iam::402087665549:role/redshift_s3_access_role", + s3_staging_location="s3://feast-int-bucket/redshift/tests/ingestion", + iam_role="arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631", workgroup="", ) environment.test_repo_config.offline_store = offline_store_config