Skip to content

Commit 26cfe7f

Browse files
ericbnericbn
authored
refactor(typing): enable boto3 implicit type annotations (aws-powertools#4692)
* chore(deps-dev): enable mypy-boto3 implicit type annotations Add boto3-stubs and botocore-stubs to enable implicit type annotations with mypy_boto3_* packages. See https://youtype.github.io/boto3_stubs_docs/mypy_boto3_s3/usage/#implicit-type-annotations This enables removing very few unneeded explicit type annotations and helps detect some new errors automatically. No need to do boto3_config = boto3_config or Config() before passing the boto3_config to the boto3_session client or resource methods, as None is an accepted value. Refactor the aws_lambda_powertools.utilities.parameters.base.BaseProvider class: move the instantiation of clients in _build_boto3_client and _build_boto3_resource_client to the subclasses and the user_agent method calls to the constructor. Keeping correct typing information with the original implementation was going to be too complex and hopefully the refactor even made the code tidier. Keep imports consistent: use boto3.session.Session, mypy_boto3_*.client, mypy_boto3_*.service_resource and mypy_boto3_*.type_defs as documented at https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html and https://youtype.github.io/boto3_stubs_docs/mypy_boto3_s3/usage/#explicit-type-annotations * Ignore mypy arg-type error with ConditionalCheckFailedException * Reassign to boto3_session instead of inlined expression
1 parent 0f610dc commit 26cfe7f

24 files changed

Lines changed: 552 additions & 251 deletions

File tree

‎aws_lambda_powertools/utilities/data_classes/s3_object_event.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ class S3ObjectLambdaEvent(DictWrapper):
220220
import requests
221221
from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent
222222
223-
session = boto3.Session()
223+
session = boto3.session.Session()
224224
s3 = session.client("s3")
225225
226226
def lambda_handler(event, context):

‎aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
)
2424

2525
if TYPE_CHECKING:
26-
from mypy_boto3_dynamodb import DynamoDBClient
26+
from mypy_boto3_dynamodb.client import DynamoDBClient
2727
from mypy_boto3_dynamodb.type_defs import AttributeValueTypeDef
2828

2929
logger = logging.getLogger(__name__)
@@ -43,7 +43,7 @@ def __init__(
4343
validation_key_attr: str = "validation",
4444
boto_config: Optional[Config] = None,
4545
boto3_session: Optional[boto3.session.Session] = None,
46-
boto3_client: "DynamoDBClient" | None = None,
46+
boto3_client: Optional[DynamoDBClient] = None,
4747
):
4848
"""
4949
Initialize the DynamoDB client
@@ -71,7 +71,7 @@ def __init__(
7171
DynamoDB attribute name for hashed representation of the parts of the event used for validation
7272
boto_config: botocore.config.Config, optional
7373
Botocore configuration to pass during client initialization
74-
boto3_session : boto3.Session, optional
74+
boto3_session : boto3.session.Session, optional
7575
Boto3 session to use for AWS API communication
7676
boto3_client : DynamoDBClient, optional
7777
Boto3 DynamoDB Client to use, boto3_session and boto_config will be ignored if both are provided
@@ -91,11 +91,9 @@ def __init__(
9191
>>> return {"StatusCode": 200}
9292
"""
9393
if boto3_client is None:
94-
self._boto_config = boto_config or Config()
95-
self._boto3_session: boto3.Session = boto3_session or boto3.session.Session()
96-
self.client: "DynamoDBClient" = self._boto3_session.client("dynamodb", config=self._boto_config)
97-
else:
98-
self.client = boto3_client
94+
boto3_session = boto3_session or boto3.session.Session()
95+
boto3_client = boto3_session.client("dynamodb", config=boto_config)
96+
self.client = boto3_client
9997

10098
user_agent.register_feature_to_client(client=self.client, feature="idempotency")
10199

@@ -246,13 +244,20 @@ def _put_record(self, data_record: DataRecord) -> None:
246244
":now_in_millis": {"N": str(int(now.timestamp() * 1000))},
247245
":inprogress": {"S": STATUS_CONSTANTS["INPROGRESS"]},
248246
},
249-
**self.return_value_on_condition, # type: ignore
247+
**self.return_value_on_condition, # type: ignore[arg-type]
250248
)
251249
except ClientError as exc:
252250
error_code = exc.response.get("Error", {}).get("Code")
253251
if error_code == "ConditionalCheckFailedException":
254-
old_data_record = self._item_to_data_record(exc.response["Item"]) if "Item" in exc.response else None
255-
if old_data_record is not None:
252+
try:
253+
item = exc.response["Item"] # type: ignore[typeddict-item]
254+
except KeyError:
255+
logger.debug(
256+
f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}",
257+
)
258+
raise IdempotencyItemAlreadyExistsError() from exc
259+
else:
260+
old_data_record = self._item_to_data_record(item)
256261
logger.debug(
257262
f"Failed to put record for already existing idempotency key: "
258263
f"{data_record.idempotency_key} with status: {old_data_record.status}, "
@@ -268,11 +273,6 @@ def _put_record(self, data_record: DataRecord) -> None:
268273

269274
raise IdempotencyItemAlreadyExistsError(old_data_record=old_data_record) from exc
270275

271-
logger.debug(
272-
f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}",
273-
)
274-
raise IdempotencyItemAlreadyExistsError() from exc
275-
276276
raise
277277

278278
@staticmethod
@@ -297,7 +297,7 @@ def boto3_supports_condition_check_failure(boto3_version: str) -> bool:
297297
def _update_record(self, data_record: DataRecord):
298298
logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}")
299299
update_expression = "SET #response_data = :response_data, #expiry = :expiry, #status = :status"
300-
expression_attr_values: Dict[str, "AttributeValueTypeDef"] = {
300+
expression_attr_values: Dict[str, AttributeValueTypeDef] = {
301301
":expiry": {"N": str(data_record.expiry_timestamp)},
302302
":response_data": {"S": data_record.response_data},
303303
":status": {"S": data_record.status},

‎aws_lambda_powertools/utilities/parameters/appconfig.py‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@
33
"""
44

55
import os
6-
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
6+
from typing import TYPE_CHECKING, Dict, Optional, Union
77

88
import boto3
99
from botocore.config import Config
1010

1111
from aws_lambda_powertools.utilities.parameters.types import TransformOptions
1212

1313
if TYPE_CHECKING:
14-
from mypy_boto3_appconfigdata import AppConfigDataClient
14+
from mypy_boto3_appconfigdata.client import AppConfigDataClient
1515

1616
from aws_lambda_powertools.shared import constants
1717
from aws_lambda_powertools.shared.functions import (
@@ -67,8 +67,6 @@ class AppConfigProvider(BaseProvider):
6767
6868
"""
6969

70-
client: Any = None
71-
7270
def __init__(
7371
self,
7472
environment: str,
@@ -80,15 +78,10 @@ def __init__(
8078
"""
8179
Initialize the App Config client
8280
"""
83-
84-
super().__init__()
85-
86-
self.client: "AppConfigDataClient" = self._build_boto3_client(
87-
service_name="appconfigdata",
88-
client=boto3_client,
89-
session=boto3_session,
90-
config=config,
91-
)
81+
if boto3_client is None:
82+
boto3_session = boto3_session or boto3.session.Session()
83+
boto3_client = boto3_session.client("appconfigdata", config=config)
84+
self.client = boto3_client
9285

9386
self.application = resolve_env_var_choice(
9487
choice=application,
@@ -99,9 +92,11 @@ def __init__(
9992

10093
self._next_token: Dict[str, str] = {} # nosec - token for get_latest_configuration executions
10194
# Dict to store the recently retrieved value for a specific configuration.
102-
self.last_returned_value: Dict[str, str] = {}
95+
self.last_returned_value: Dict[str, bytes] = {}
96+
97+
super().__init__(client=self.client)
10398

104-
def _get(self, name: str, **sdk_options) -> str:
99+
def _get(self, name: str, **sdk_options) -> bytes:
105100
"""
106101
Retrieve a parameter value from AWS App config.
107102

‎aws_lambda_powertools/utilities/parameters/base.py‎

Lines changed: 5 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -10,43 +10,30 @@
1010
from abc import ABC, abstractmethod
1111
from datetime import datetime, timedelta
1212
from typing import (
13-
TYPE_CHECKING,
1413
Any,
1514
Callable,
1615
Dict,
1716
NamedTuple,
1817
Optional,
1918
Tuple,
20-
Type,
2119
Union,
2220
cast,
2321
overload,
2422
)
2523

26-
import boto3
27-
from botocore.config import Config
28-
2924
from aws_lambda_powertools.shared import constants, user_agent
3025
from aws_lambda_powertools.shared.functions import resolve_max_age
3126
from aws_lambda_powertools.utilities.parameters.types import TransformOptions
3227

3328
from .exceptions import GetParameterError, TransformParameterError
3429

35-
if TYPE_CHECKING:
36-
from mypy_boto3_appconfigdata import AppConfigDataClient
37-
from mypy_boto3_dynamodb import DynamoDBServiceResource
38-
from mypy_boto3_secretsmanager import SecretsManagerClient
39-
from mypy_boto3_ssm import SSMClient
40-
41-
4230
DEFAULT_MAX_AGE_SECS = "300"
4331

4432
# These providers will be dynamically initialized on first use of the helper functions
4533
DEFAULT_PROVIDERS: Dict[str, Any] = {}
4634
TRANSFORM_METHOD_JSON = "json"
4735
TRANSFORM_METHOD_BINARY = "binary"
4836
SUPPORTED_TRANSFORM_METHODS = [TRANSFORM_METHOD_JSON, TRANSFORM_METHOD_BINARY]
49-
ParameterClients = Union["AppConfigDataClient", "SecretsManagerClient", "SSMClient"]
5037

5138
TRANSFORM_METHOD_MAPPING = {
5239
TRANSFORM_METHOD_JSON: json.loads,
@@ -69,10 +56,14 @@ class BaseProvider(ABC):
6956

7057
store: Dict[Tuple, ExpirableValue]
7158

72-
def __init__(self):
59+
def __init__(self, *, client=None, resource=None):
7360
"""
7461
Initialize the base provider
7562
"""
63+
if client is not None:
64+
user_agent.register_feature_to_client(client=client, feature="parameters")
65+
if resource is not None:
66+
user_agent.register_feature_to_resource(resource=resource, feature="parameters")
7667

7768
self.store: Dict[Tuple, ExpirableValue] = {}
7869

@@ -262,78 +253,6 @@ def _build_cache_key(
262253
"""
263254
return (name, transform, is_nested)
264255

265-
@staticmethod
266-
def _build_boto3_client(
267-
service_name: str,
268-
client: Optional[ParameterClients] = None,
269-
session: Optional[Type[boto3.Session]] = None,
270-
config: Optional[Type[Config]] = None,
271-
) -> Type[ParameterClients]:
272-
"""Builds a low level boto3 client with session and config provided
273-
274-
Parameters
275-
----------
276-
service_name : str
277-
AWS service name to instantiate a boto3 client, e.g. ssm
278-
client : Optional[ParameterClients], optional
279-
boto3 client instance, by default None
280-
session : Optional[Type[boto3.Session]], optional
281-
boto3 session instance, by default None
282-
config : Optional[Type[Config]], optional
283-
botocore config instance to configure client with, by default None
284-
285-
Returns
286-
-------
287-
Type[ParameterClients]
288-
Instance of a boto3 client for Parameters feature (e.g., ssm, appconfig, secretsmanager, etc.)
289-
"""
290-
if client is not None:
291-
user_agent.register_feature_to_client(client=client, feature="parameters")
292-
return client
293-
294-
session = session or boto3.Session()
295-
config = config or Config()
296-
client = session.client(service_name=service_name, config=config)
297-
user_agent.register_feature_to_client(client=client, feature="parameters")
298-
return client
299-
300-
# maintenance: change DynamoDBServiceResource type to ParameterResourceClients when we expand
301-
@staticmethod
302-
def _build_boto3_resource_client(
303-
service_name: str,
304-
client: Optional["DynamoDBServiceResource"] = None,
305-
session: Optional[Type[boto3.Session]] = None,
306-
config: Optional[Type[Config]] = None,
307-
endpoint_url: Optional[str] = None,
308-
) -> "DynamoDBServiceResource":
309-
"""Builds a high level boto3 resource client with session, config and endpoint_url provided
310-
311-
Parameters
312-
----------
313-
service_name : str
314-
AWS service name to instantiate a boto3 client, e.g. ssm
315-
client : Optional[DynamoDBServiceResource], optional
316-
boto3 client instance, by default None
317-
session : Optional[Type[boto3.Session]], optional
318-
boto3 session instance, by default None
319-
config : Optional[Type[Config]], optional
320-
botocore config instance to configure client, by default None
321-
322-
Returns
323-
-------
324-
Type[DynamoDBServiceResource]
325-
Instance of a boto3 resource client for Parameters feature (e.g., dynamodb, etc.)
326-
"""
327-
if client is not None:
328-
user_agent.register_feature_to_resource(resource=client, feature="parameters")
329-
return client
330-
331-
session = session or boto3.Session()
332-
config = config or Config()
333-
client = session.resource(service_name=service_name, config=config, endpoint_url=endpoint_url)
334-
user_agent.register_feature_to_resource(resource=client, feature="parameters")
335-
return client
336-
337256

338257
def get_transform_method(value: str, transform: TransformOptions = None) -> Callable[..., Any]:
339258
"""

‎aws_lambda_powertools/utilities/parameters/dynamodb.py‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@
1111
from .base import BaseProvider
1212

1313
if TYPE_CHECKING:
14-
from mypy_boto3_dynamodb import DynamoDBServiceResource
15-
from mypy_boto3_dynamodb.service_resource import Table
14+
from mypy_boto3_dynamodb.service_resource import DynamoDBServiceResource
1615

1716

1817
class DynamoDBProvider(BaseProvider):
@@ -162,19 +161,15 @@ def __init__(
162161
"""
163162
Initialize the DynamoDB client
164163
"""
165-
self.table: "Table" = self._build_boto3_resource_client(
166-
service_name="dynamodb",
167-
client=boto3_client,
168-
session=boto3_session,
169-
config=config,
170-
endpoint_url=endpoint_url,
171-
).Table(table_name)
172-
164+
if boto3_client is None:
165+
boto3_session = boto3_session or boto3.session.Session()
166+
boto3_client = boto3_session.resource("dynamodb", config=config, endpoint_url=endpoint_url)
167+
self.table = boto3_client.Table(table_name)
173168
self.key_attr = key_attr
174169
self.sort_attr = sort_attr
175170
self.value_attr = value_attr
176171

177-
super().__init__()
172+
super().__init__(resource=boto3_client)
178173

179174
def _get(self, name: str, **sdk_options) -> str:
180175
"""

0 commit comments

Comments
 (0)