forked from aws/aws-cli
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathpaginate.py
More file actions
297 lines (251 loc) · 11.7 KB
/
paginate.py
File metadata and controls
297 lines (251 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""This module has customizations to unify paging parameters.
For any operation that can be paginated, we will:
* Hide the service specific pagination params. This can vary across
services and we're going to replace them with a consistent set of
arguments. The arguments will still work, but they are not
documented. This allows us to add a pagination config after
the fact and still remain backwards compatible with users that
were manually doing pagination.
* Add a ``--starting-token`` and a ``--max-items`` argument.
"""
import logging
from functools import partial
from botocore import xform_name
from botocore.exceptions import DataNotFoundError, PaginationError
from botocore import model
from awscli.arguments import BaseCLIArgument
logger = logging.getLogger(__name__)
STARTING_TOKEN_HELP = """
<p>A token to specify where to start paginating. This is the
<code>NextToken</code> from a previously truncated response.</p>
<p>For usage examples, see <a
href="https://docs.aws.amazon.com/cli/latest/userguide/pagination.html"
>Pagination</a> in the <i>AWS Command Line Interface User
Guide</i>.</p>
"""
MAX_ITEMS_HELP = """
<p>The total number of items to return in the command's output.
If the total number of items available is more than the value
specified, a <code>NextToken</code> is provided in the command's
output. To resume pagination, provide the
<code>NextToken</code> value in the <code>starting-token</code>
argument of a subsequent command. <b>Do not</b> use the
<code>NextToken</code> response element directly outside of the
AWS CLI.</p>
<p>For usage examples, see <a
href="https://docs.aws.amazon.com/cli/latest/userguide/pagination.html"
>Pagination</a> in the <i>AWS Command Line Interface User
Guide</i>.</p>
"""
PAGE_SIZE_HELP = """
<p>The size of each page to get in the AWS service call. This
does not affect the number of items returned in the command's
output. Setting a smaller page size results in more calls to
the AWS service, retrieving fewer items in each call. This can
help prevent the AWS service calls from timing out.</p>
<p>For usage examples, see <a
href="https://docs.aws.amazon.com/cli/latest/userguide/pagination.html"
>Pagination</a> in the <i>AWS Command Line Interface User
Guide</i>.</p>
"""
def register_pagination(event_handlers):
event_handlers.register('building-argument-table', unify_paging_params)
event_handlers.register_last('doc-description', add_paging_description)
def get_paginator_config(session, service_name, operation_name):
try:
paginator_model = session.get_paginator_model(service_name)
except DataNotFoundError:
return None
try:
operation_paginator_config = paginator_model.get_paginator(
operation_name)
except ValueError:
return None
return operation_paginator_config
def add_paging_description(help_command, **kwargs):
# This customization is only applied to the description of
# Operations, so we must filter out all other events.
if not isinstance(help_command.obj, model.OperationModel):
return
service_name = help_command.obj.service_model.service_name
paginator_config = get_paginator_config(
help_command.session, service_name, help_command.obj.name)
if not paginator_config:
return
help_command.doc.style.new_paragraph()
help_command.doc.writeln(
('``%s`` is a paginated operation. Multiple API calls may be issued '
'in order to retrieve the entire data set of results. You can '
'disable pagination by providing the ``--no-paginate`` argument.')
% help_command.name)
# Only include result key information if it is present.
if paginator_config.get('result_key'):
queries = paginator_config['result_key']
if type(queries) is not list:
queries = [queries]
queries = ", ".join([('``%s``' % s) for s in queries])
help_command.doc.writeln(
('When using ``--output text`` and the ``--query`` argument on a '
'paginated response, the ``--query`` argument must extract data '
'from the results of the following query expressions: %s')
% queries)
def unify_paging_params(argument_table, operation_model, event_name,
session, **kwargs):
paginator_config = get_paginator_config(
session, operation_model.service_model.service_name,
operation_model.name)
if paginator_config is None:
# We only apply these customizations to paginated responses.
return
logger.debug("Modifying paging parameters for operation: %s",
operation_model.name)
_remove_existing_paging_arguments(argument_table, paginator_config)
parsed_args_event = event_name.replace('building-argument-table.',
'operation-args-parsed.')
shadowed_args = {}
add_paging_argument(argument_table, 'starting-token',
PageArgument('starting-token', STARTING_TOKEN_HELP,
parse_type='string',
serialized_name='StartingToken'),
shadowed_args)
input_members = operation_model.input_shape.members
type_name = 'integer'
if 'limit_key' in paginator_config:
limit_key_shape = input_members[paginator_config['limit_key']]
type_name = limit_key_shape.type_name
if type_name not in PageArgument.type_map:
raise TypeError(
('Unsupported pagination type {0} for operation {1}'
' and parameter {2}').format(
type_name, operation_model.name,
paginator_config['limit_key']))
add_paging_argument(argument_table, 'page-size',
PageArgument('page-size', PAGE_SIZE_HELP,
parse_type=type_name,
serialized_name='PageSize'),
shadowed_args)
add_paging_argument(argument_table, 'max-items',
PageArgument('max-items', MAX_ITEMS_HELP,
parse_type=type_name,
serialized_name='MaxItems'),
shadowed_args)
session.register(
parsed_args_event,
partial(check_should_enable_pagination,
list(_get_all_cli_input_tokens(paginator_config)),
shadowed_args, argument_table))
def add_paging_argument(argument_table, arg_name, argument, shadowed_args):
if arg_name in argument_table:
# If there's already an entry in the arg table for this argument,
# this means we're shadowing an argument for this operation. We
# need to store this later in case pagination is turned off because
# we put these arguments back.
# See the comment in check_should_enable_pagination() for more info.
shadowed_args[arg_name] = argument_table[arg_name]
argument_table[arg_name] = argument
def check_should_enable_pagination(input_tokens, shadowed_args, argument_table,
parsed_args, parsed_globals, **kwargs):
normalized_paging_args = ['start_token', 'max_items']
for token in input_tokens:
py_name = token.replace('-', '_')
if getattr(parsed_args, py_name) is not None and \
py_name not in normalized_paging_args:
# The user has specified a manual (undocumented) pagination arg.
# We need to automatically turn pagination off.
logger.debug("User has specified a manual pagination arg. "
"Automatically setting --no-paginate.")
parsed_globals.paginate = False
if not parsed_globals.paginate:
ensure_paging_params_not_set(parsed_args, shadowed_args)
# Because pagination is now disabled, there's a chance that
# we were shadowing arguments. For example, we inject a
# --max-items argument in unify_paging_params(). If the
# the operation also provides its own MaxItems (which we
# expose as --max-items) then our custom pagination arg
# was shadowing the customers arg. When we turn pagination
# off we need to put back the original argument which is
# what we're doing here.
for key, value in shadowed_args.items():
argument_table[key] = value
def ensure_paging_params_not_set(parsed_args, shadowed_args):
paging_params = ['starting_token', 'page_size', 'max_items']
shadowed_params = [p.replace('-', '_') for p in shadowed_args.keys()]
params_used = [p for p in paging_params if
p not in shadowed_params and getattr(parsed_args, p, None)]
if len(params_used) > 0:
converted_params = ', '.join(
["--" + p.replace('_', '-') for p in params_used])
raise PaginationError(
message="Cannot specify --no-paginate along with pagination "
"arguments: %s" % converted_params)
def _remove_existing_paging_arguments(argument_table, pagination_config):
for cli_name in _get_all_cli_input_tokens(pagination_config):
argument_table[cli_name]._UNDOCUMENTED = True
def _get_all_cli_input_tokens(pagination_config):
# Get all input tokens including the limit_key
# if it exists.
tokens = _get_input_tokens(pagination_config)
for token_name in tokens:
cli_name = xform_name(token_name, '-')
yield cli_name
if 'limit_key' in pagination_config:
key_name = pagination_config['limit_key']
cli_name = xform_name(key_name, '-')
yield cli_name
def _get_input_tokens(pagination_config):
tokens = pagination_config['input_token']
if not isinstance(tokens, list):
return [tokens]
return tokens
def _get_cli_name(param_objects, token_name):
for param in param_objects:
if param.name == token_name:
return param.cli_name.lstrip('-')
class PageArgument(BaseCLIArgument):
type_map = {
'string': str,
'integer': int,
'long': int,
}
def __init__(self, name, documentation, parse_type, serialized_name):
self.argument_model = model.Shape('PageArgument', {'type': 'string'})
self._name = name
self._serialized_name = serialized_name
self._documentation = documentation
self._parse_type = parse_type
self._required = False
@property
def cli_name(self):
return '--' + self._name
@property
def cli_type_name(self):
return self._parse_type
@property
def required(self):
return self._required
@required.setter
def required(self, value):
self._required = value
@property
def documentation(self):
return self._documentation
def add_to_parser(self, parser):
parser.add_argument(self.cli_name, dest=self.py_name,
type=self.type_map[self._parse_type])
def add_to_params(self, parameters, value):
if value is not None:
pagination_config = parameters.get('PaginationConfig', {})
pagination_config[self._serialized_name] = value
parameters['PaginationConfig'] = pagination_config