Skip to content

Commit 29a797b

Browse files
kyleknapkyleknap
committed
Merge pull request aws#872 from kyleknap/source-region
Added a ``--source-region`` parameter.
2 parents a6a3c2d + 8e832a6 commit 29a797b

16 files changed

Lines changed: 464 additions & 216 deletions

‎awscli/customizations/s3/filegenerator.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
from dateutil.parser import parse
1818
from dateutil.tz import tzlocal
1919

20-
from awscli.customizations.s3.fileinfo import FileInfo
2120
from awscli.customizations.s3.utils import find_bucket_key, get_file_stat
2221
from awscli.customizations.s3.utils import BucketLister
2322
from awscli.errorhandler import ClientError
@@ -46,6 +45,20 @@ def __init__(self, directory, filename):
4645
super(FileDecodingError, self).__init__(self.error_message)
4746

4847

48+
class FileStat(object):
49+
def __init__(self, src, dest=None, compare_key=None, size=None,
50+
last_update=None, src_type=None, dest_type=None,
51+
operation_name=None):
52+
self.src = src
53+
self.dest = dest
54+
self.compare_key = compare_key
55+
self.size = size
56+
self.last_update = last_update
57+
self.src_type = src_type
58+
self.dest_type = dest_type
59+
self.operation_name = operation_name
60+
61+
4962
class FileGenerator(object):
5063
"""
5164
This is a class the creates a generator to yield files based on information
@@ -54,7 +67,8 @@ class FileGenerator(object):
5467
under the same common prefix. The generator yields corresponding
5568
``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
5669
"""
57-
def __init__(self, service, endpoint, operation_name, follow_symlinks=True):
70+
def __init__(self, service, endpoint, operation_name,
71+
follow_symlinks=True):
5872
self._service = service
5973
self._endpoint = endpoint
6074
self.operation_name = operation_name
@@ -86,10 +100,9 @@ def call(self, files):
86100
sep_table[dest_type])
87101
else:
88102
dest_path = dest['path']
89-
yield FileInfo(src=src_path, dest=dest_path,
103+
yield FileStat(src=src_path, dest=dest_path,
90104
compare_key=compare_key, size=size,
91105
last_update=last_update, src_type=src_type,
92-
service=self._service, endpoint=self._endpoint,
93106
dest_type=dest_type,
94107
operation_name=self.operation_name)
95108

‎awscli/customizations/s3/fileinfo.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ class FileInfo(TaskInfo):
140140
def __init__(self, src, dest=None, compare_key=None, size=None,
141141
last_update=None, src_type=None, dest_type=None,
142142
operation_name=None, service=None, endpoint=None,
143-
parameters=None):
143+
parameters=None, source_endpoint=None):
144144
super(FileInfo, self).__init__(src, src_type=src_type,
145145
operation_name=operation_name,
146146
service=service,
@@ -156,6 +156,7 @@ def __init__(self, src, dest=None, compare_key=None, size=None,
156156
else:
157157
self.parameters = {'acl': None,
158158
'sse': None}
159+
self.source_endpoint = source_endpoint
159160

160161
def _permission_to_param(self, permission):
161162
if permission == 'read':
@@ -256,7 +257,8 @@ def delete(self):
256257
"""
257258
if (self.src_type == 's3'):
258259
bucket, key = find_bucket_key(self.src)
259-
params = {'endpoint': self.endpoint, 'bucket': bucket, 'key': key}
260+
params = {'endpoint': self.source_endpoint, 'bucket': bucket,
261+
'key': key}
260262
response_data, http = operate(self.service, 'DeleteObject',
261263
params)
262264
else:

‎awscli/customizations/s3/fileinfobuilder.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
from awscli.customizations.s3.fileinfo import FileInfo
14+
15+
16+
class FileInfoBuilder(object):
17+
"""
18+
This class takes a ``FileBase`` object's attributes and generates
19+
a ``FileInfo`` object so that the operation can be performed.
20+
"""
21+
def __init__(self, service, endpoint, source_endpoint=None,
22+
parameters = None):
23+
self._service = service
24+
self._endpoint = endpoint
25+
self._source_endpoint = endpoint
26+
if source_endpoint:
27+
self._source_endpoint = source_endpoint
28+
self._parameters = parameters
29+
30+
def call(self, files):
31+
for file_base in files:
32+
file_info = self._inject_info(file_base)
33+
yield file_info
34+
35+
def _inject_info(self, file_base):
36+
file_info_attr = {}
37+
file_info_attr['src'] = file_base.src
38+
file_info_attr['dest'] = file_base.dest
39+
file_info_attr['compare_key'] = file_base.compare_key
40+
file_info_attr['size'] = file_base.size
41+
file_info_attr['last_update'] = file_base.last_update
42+
file_info_attr['src_type'] = file_base.src_type
43+
file_info_attr['dest_type'] = file_base.dest_type
44+
file_info_attr['operation_name'] = file_base.operation_name
45+
file_info_attr['service'] = self._service
46+
file_info_attr['endpoint'] = self._endpoint
47+
file_info_attr['source_endpoint'] = self._source_endpoint
48+
file_info_attr['parameters'] = self._parameters
49+
return FileInfo(**file_info_attr)

‎awscli/customizations/s3/subcommands.py‎

Lines changed: 51 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
from awscli.customizations.commands import BasicCommand
2121
from awscli.customizations.s3.comparator import Comparator
22+
from awscli.customizations.s3.fileinfobuilder import FileInfoBuilder
2223
from awscli.customizations.s3.fileformat import FileFormat
2324
from awscli.customizations.s3.filegenerator import FileGenerator
2425
from awscli.customizations.s3.fileinfo import TaskInfo
@@ -163,6 +164,16 @@
163164
CONTENT_LANGUAGE = {'name': 'content-language', 'nargs': 1,
164165
'help_text': ("The language the content is in.")}
165166

167+
SOURCE_REGION = {'name': 'source-region', 'nargs': 1,
168+
'help_text': (
169+
"When transferring objects from an s3 bucket to an s3 "
170+
"bucket, this specifies the region of the source bucket."
171+
" Note the region specified by ``--region`` or through "
172+
"configuration of the CLI refers to the region of the "
173+
"destination bucket. If ``--source-region`` is not "
174+
"specified the region of the source will be the same "
175+
"as the region of the destination bucket.")}
176+
166177
EXPIRES = {'name': 'expires', 'nargs': 1, 'help_text': ("The date and time at "
167178
"which the object is no longer cacheable.")}
168179

@@ -198,20 +209,22 @@
198209
FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
199210
SSE, STORAGE_CLASS, GRANTS, WEBSITE_REDIRECT, CONTENT_TYPE,
200211
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING,
201-
CONTENT_LANGUAGE, EXPIRES]
212+
CONTENT_LANGUAGE, EXPIRES, SOURCE_REGION]
202213

203214
SYNC_ARGS = [DELETE, EXACT_TIMESTAMPS, SIZE_ONLY] + TRANSFER_ARGS
204215

205216

217+
def get_endpoint(service, region, endpoint_url, verify):
218+
return service.get_endpoint(region_name=region, endpoint_url=endpoint_url,
219+
verify=verify)
220+
221+
206222
class S3Command(BasicCommand):
207223
def _run_main(self, parsed_args, parsed_globals):
208224
self.service = self._session.get_service('s3')
209-
self.endpoint = self._get_endpoint(self.service, parsed_globals)
210-
211-
def _get_endpoint(self, service, parsed_globals):
212-
return service.get_endpoint(region_name=parsed_globals.region,
213-
endpoint_url=parsed_globals.endpoint_url,
214-
verify=parsed_globals.verify_ssl)
225+
self.endpoint = get_endpoint(self.service, parsed_globals.region,
226+
parsed_globals.endpoint_url,
227+
parsed_globals.verify_ssl)
215228

216229

217230
class ListCommand(S3Command):
@@ -363,6 +376,7 @@ def _run_main(self, parsed_args, parsed_globals):
363376
cmd_params.check_force(parsed_globals)
364377
cmd = CommandArchitecture(self._session, self.NAME,
365378
cmd_params.parameters)
379+
cmd.set_endpoints()
366380
cmd.create_instructions()
367381
return cmd.run()
368382

@@ -463,10 +477,25 @@ def __init__(self, session, cmd, parameters):
463477
self.parameters = parameters
464478
self.instructions = []
465479
self._service = self.session.get_service('s3')
466-
self._endpoint = self._service.get_endpoint(
467-
region_name=self.parameters['region'],
480+
self._endpoint = None
481+
self._source_endpoint = None
482+
483+
def set_endpoints(self):
484+
self._endpoint = get_endpoint(
485+
self._service,
486+
region=self.parameters['region'],
468487
endpoint_url=self.parameters['endpoint_url'],
469-
verify=self.parameters['verify_ssl'])
488+
verify=self.parameters['verify_ssl']
489+
)
490+
self._source_endpoint = self._endpoint
491+
if self.parameters['source_region']:
492+
if self.parameters['paths_type'] == 's3s3':
493+
self._source_endpoint = get_endpoint(
494+
self._service,
495+
region=self.parameters['source_region'][0],
496+
endpoint_url=None,
497+
verify=self.parameters['verify_ssl']
498+
)
470499

471500
def create_instructions(self):
472501
"""
@@ -482,6 +511,8 @@ def create_instructions(self):
482511
self.instructions.append('filters')
483512
if self.cmd == 'sync':
484513
self.instructions.append('comparator')
514+
if self.cmd not in ['mb', 'rb']:
515+
self.instructions.append('file_info_builder')
485516
self.instructions.append('s3_handler')
486517

487518
def run(self):
@@ -524,7 +555,8 @@ def run(self):
524555
'rb': 'remove_bucket'
525556
}
526557
operation_name = cmd_translation[paths_type][self.cmd]
527-
file_generator = FileGenerator(self._service, self._endpoint,
558+
file_generator = FileGenerator(self._service,
559+
self._source_endpoint,
528560
operation_name,
529561
self.parameters['follow_symlinks'])
530562
rev_generator = FileGenerator(self._service, self._endpoint, '',
@@ -534,6 +566,8 @@ def run(self):
534566
operation_name=operation_name,
535567
service=self._service,
536568
endpoint=self._endpoint)]
569+
file_info_builder = FileInfoBuilder(self._service, self._endpoint,
570+
self._source_endpoint, self.parameters)
537571
s3handler = S3Handler(self.session, self.parameters)
538572

539573
command_dict = {}
@@ -544,21 +578,25 @@ def run(self):
544578
'filters': [create_filter(self.parameters),
545579
create_filter(self.parameters)],
546580
'comparator': [Comparator(self.parameters)],
581+
'file_info_builder': [file_info_builder],
547582
's3_handler': [s3handler]}
548583
elif self.cmd == 'cp':
549584
command_dict = {'setup': [files],
550585
'file_generator': [file_generator],
551586
'filters': [create_filter(self.parameters)],
587+
'file_info_builder': [file_info_builder],
552588
's3_handler': [s3handler]}
553589
elif self.cmd == 'rm':
554590
command_dict = {'setup': [files],
555591
'file_generator': [file_generator],
556592
'filters': [create_filter(self.parameters)],
593+
'file_info_builder': [file_info_builder],
557594
's3_handler': [s3handler]}
558595
elif self.cmd == 'mv':
559596
command_dict = {'setup': [files],
560597
'file_generator': [file_generator],
561598
'filters': [create_filter(self.parameters)],
599+
'file_info_builder': [file_info_builder],
562600
's3_handler': [s3handler]}
563601
elif self.cmd == 'mb':
564602
command_dict = {'setup': [taskinfo],
@@ -610,6 +648,8 @@ def __init__(self, session, cmd, parameters, usage):
610648
self.parameters['dir_op'] = False
611649
if 'follow_symlinks' not in parameters:
612650
self.parameters['follow_symlinks'] = True
651+
if 'source_region' not in parameters:
652+
self.parameters['source_region'] = None
613653
if self.cmd in ['sync', 'mb', 'rb']:
614654
self.parameters['dir_op'] = True
615655

‎awscli/testutils.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,9 @@ def create_file(self, filename, contents, mtime=None):
333333
os.makedirs(os.path.dirname(full_path))
334334
with open(full_path, 'w') as f:
335335
f.write(contents)
336+
current_time = os.path.getmtime(full_path)
337+
# Subtract a few years off the last modification date.
338+
os.utime(full_path, (current_time, current_time - 100000000))
336339
if mtime is not None:
337340
os.utime(full_path, (mtime, mtime))
338341
return full_path

‎tests/integration/customizations/s3/test_filegenerator.py‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@
2222

2323
import botocore.session
2424
from awscli import EnvironmentVariables
25-
from awscli.customizations.s3.filegenerator import FileGenerator
26-
from awscli.customizations.s3.fileinfo import FileInfo
25+
from awscli.customizations.s3.filegenerator import FileGenerator, FileStat
2726
from tests.unit.customizations.s3 import make_s3_files, s3_cleanup, \
2827
compare_files
2928

@@ -52,14 +51,14 @@ def test_s3_file(self):
5251
result_list = list(
5352
FileGenerator(self.service, self.endpoint, '').call(
5453
input_s3_file))
55-
file_info = FileInfo(src=self.file1, dest='text1.txt',
54+
file_stat = FileStat(src=self.file1, dest='text1.txt',
5655
compare_key='text1.txt',
5756
size=expected_file_size,
5857
last_update=result_list[0].last_update,
5958
src_type='s3',
6059
dest_type='local', operation_name='')
6160

62-
expected_list = [file_info]
61+
expected_list = [file_stat]
6362
self.assertEqual(len(result_list), 1)
6463
compare_files(self, result_list[0], expected_list[0])
6564

@@ -75,22 +74,22 @@ def test_s3_directory(self):
7574
result_list = list(
7675
FileGenerator(self.service, self.endpoint, '').call(
7776
input_s3_file))
78-
file_info = FileInfo(src=self.file2,
77+
file_stat = FileStat(src=self.file2,
7978
dest='another_directory' + os.sep + 'text2.txt',
8079
compare_key='another_directory/text2.txt',
8180
size=21,
8281
last_update=result_list[0].last_update,
8382
src_type='s3',
8483
dest_type='local', operation_name='')
85-
file_info2 = FileInfo(src=self.file1,
84+
file_stat2 = FileStat(src=self.file1,
8685
dest='text1.txt',
8786
compare_key='text1.txt',
8887
size=15,
8988
last_update=result_list[1].last_update,
9089
src_type='s3',
9190
dest_type='local', operation_name='')
9291

93-
expected_result = [file_info, file_info2]
92+
expected_result = [file_stat, file_stat2]
9493
self.assertEqual(len(result_list), 2)
9594
compare_files(self, result_list[0], expected_result[0])
9695
compare_files(self, result_list[1], expected_result[1])
@@ -109,37 +108,32 @@ def test_s3_delete_directory(self):
109108
'delete').call(
110109
input_s3_file))
111110

112-
file_info1 = FileInfo(
111+
file_stat1 = FileStat(
113112
src=self.bucket + '/another_directory/',
114113
dest='another_directory' + os.sep,
115114
compare_key='another_directory/',
116115
size=0,
117116
last_update=result_list[0].last_update,
118117
src_type='s3',
119-
dest_type='local', operation_name='delete',
120-
service=self.service, endpoint=self.endpoint)
121-
file_info2 = FileInfo(
118+
dest_type='local', operation_name='delete')
119+
file_stat2 = FileStat(
122120
src=self.file2,
123121
dest='another_directory' + os.sep + 'text2.txt',
124122
compare_key='another_directory/text2.txt',
125123
size=21,
126124
last_update=result_list[1].last_update,
127125
src_type='s3',
128-
dest_type='local', operation_name='delete',
129-
service=self.service,
130-
endpoint=self.endpoint)
131-
file_info3 = FileInfo(
126+
dest_type='local', operation_name='delete')
127+
file_stat3 = FileStat(
132128
src=self.file1,
133129
dest='text1.txt',
134130
compare_key='text1.txt',
135131
size=15,
136132
last_update=result_list[2].last_update,
137133
src_type='s3',
138-
dest_type='local', operation_name='delete',
139-
service=self.service,
140-
endpoint=self.endpoint)
134+
dest_type='local', operation_name='delete')
141135

142-
expected_list = [file_info1, file_info2, file_info3]
136+
expected_list = [file_stat1, file_stat2, file_stat3]
143137
self.assertEqual(len(result_list), 3)
144138
compare_files(self, result_list[0], expected_list[0])
145139
compare_files(self, result_list[1], expected_list[1])

0 commit comments

Comments
 (0)