From e604f61d85e4d5081a31873c7f9d3fa77ff0fcb5 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Fri, 13 Oct 2017 17:25:38 +0200 Subject: [PATCH 1/3] Add initial version Signed-off-by: Jayson Reis --- .editorconfig | 3 + .travis.yml | 9 --- appveyor.yml | 28 +------- ci/templates/appveyor.yml | 6 +- setup.py | 9 ++- src/efs/__init__.py | 4 ++ src/efs/efosfs.py | 8 +++ src/efs/efs3.py | 58 ++++++++++++++++ src/efs/filesystem.py | 122 +++++++++++++++++++++++++++++++++ tests/conftest.py | 61 +++++++++++++++++ tests/test_efs.py | 6 -- tests/test_local_filesystem.py | 94 +++++++++++++++++++++++++ tests/test_s3_filesystem.py | 111 ++++++++++++++++++++++++++++++ tox.ini | 9 ++- 14 files changed, 475 insertions(+), 53 deletions(-) create mode 100644 src/efs/efosfs.py create mode 100644 src/efs/efs3.py create mode 100644 src/efs/filesystem.py create mode 100644 tests/conftest.py delete mode 100644 tests/test_efs.py create mode 100644 tests/test_local_filesystem.py create mode 100644 tests/test_s3_filesystem.py diff --git a/.editorconfig b/.editorconfig index 4000618..4a1fb8d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,3 +11,6 @@ charset = utf-8 [*.{bat,cmd,ps1}] end_of_line = crlf + +[*.{yml}] +indent_size = 2 diff --git a/.travis.yml b/.travis.yml index 1a9adea..1abe0e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,6 @@ env: - TOXENV=docs matrix: include: - - python: '2.7' - env: - - TOXENV=py27,report - - python: '3.3' - env: - - TOXENV=py33,report - python: '3.4' env: - TOXENV=py34,report @@ -25,9 +19,6 @@ matrix: - python: '3.6' env: - TOXENV=py36,report - - python: 'pypy-5.4' - env: - - TOXENV=pypy,report before_install: - python --version - uname -a diff --git a/appveyor.yml b/appveyor.yml index 4589f20..3719986 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,32 +7,10 @@ environment: WITH_COMPILER: 'cmd /E:ON /V:ON /C .\ci\appveyor-with-compiler.cmd' matrix: - TOXENV: check - TOXPYTHON: C:\Python27\python.exe - PYTHON_HOME: C:\Python27 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '32' - - TOXENV: 'py27,report' - TOXPYTHON: C:\Python27\python.exe - PYTHON_HOME: C:\Python27 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '32' - - TOXENV: 'py27,report' - TOXPYTHON: C:\Python27-x64\python.exe - WINDOWS_SDK_VERSION: v7.0 - PYTHON_HOME: C:\Python27-x64 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '64' - - TOXENV: 'py33,report' - TOXPYTHON: C:\Python33\python.exe - PYTHON_HOME: C:\Python33 - PYTHON_VERSION: '3.3' + TOXPYTHON: C:\Python34\python.exe + PYTHON_HOME: C:\Python34 + PYTHON_VERSION: '3.4' PYTHON_ARCH: '32' - - TOXENV: 'py33,report' - TOXPYTHON: C:\Python33-x64\python.exe - WINDOWS_SDK_VERSION: v7.1 - PYTHON_HOME: C:\Python33-x64 - PYTHON_VERSION: '3.3' - PYTHON_ARCH: '64' - TOXENV: 'py34,report' TOXPYTHON: C:\Python34\python.exe PYTHON_HOME: C:\Python34 diff --git a/ci/templates/appveyor.yml b/ci/templates/appveyor.yml index 4d440ff..c89a022 100644 --- a/ci/templates/appveyor.yml +++ b/ci/templates/appveyor.yml @@ -7,9 +7,9 @@ environment: WITH_COMPILER: 'cmd /E:ON /V:ON /C .\ci\appveyor-with-compiler.cmd' matrix: - TOXENV: check - TOXPYTHON: C:\Python27\python.exe - PYTHON_HOME: C:\Python27 - PYTHON_VERSION: '2.7' + TOXPYTHON: C:\Python34\python.exe + PYTHON_HOME: C:\Python34 + PYTHON_VERSION: '3.4' PYTHON_ARCH: '32' {% for env in tox_environments %}{{ '' }}{% if env.startswith(('py2', 'py3')) %} - TOXENV: '{{ env }},report' diff --git a/setup.py b/setup.py index dc8cfb5..7823976 100644 --- a/setup.py +++ b/setup.py @@ -56,17 +56,16 @@ def read(*names, **kwargs): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', - # uncomment if you test on these interpreters: - # 'Programming Language :: Python :: Implementation :: IronPython', - # 'Programming Language :: Python :: Implementation :: Jython', - # 'Programming Language :: Python :: Implementation :: Stackless', 'Topic :: Utilities', ], keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], install_requires=[ - # eg: 'aspectlib==1.1.1', 'six>=1.7', + 'Flask', + 'fs >= 0.5.4, < 2.0', + 'autorepr', + 'boto' ], extras_require={ # eg: diff --git a/src/efs/__init__.py b/src/efs/__init__.py index 3dc1f76..67ce475 100644 --- a/src/efs/__init__.py +++ b/src/efs/__init__.py @@ -1 +1,5 @@ __version__ = "0.1.0" + +from .filesystem import EFS + +__all__ = ('EFS', ) diff --git a/src/efs/efosfs.py b/src/efs/efosfs.py new file mode 100644 index 0000000..891418d --- /dev/null +++ b/src/efs/efosfs.py @@ -0,0 +1,8 @@ +from autorepr import autorepr +from fs.osfs import OSFS + + +class EFOSFS(OSFS): + """Simple wrapper to have a better repr.""" + + __repr__ = autorepr(['root_path', 'dir_mode:o']) diff --git a/src/efs/efs3.py b/src/efs/efs3.py new file mode 100644 index 0000000..e24d299 --- /dev/null +++ b/src/efs/efs3.py @@ -0,0 +1,58 @@ +"""Eatfirst FileSystem for S3.""" + +from autorepr import autorepr +from fs.path import iteratepath +from fs.path import normpath +from fs.path import relpath +from fs.s3fs import S3FS +from fs.s3fs import thread_local + + +class EFS3(S3FS): + """Extension of S3FS class, fixing it to work with python 3k.""" + + def __init__(self, bucket, prefix='', aws_access_key=None, aws_secret_key=None, separator='/', + thread_synchronize=True, key_sync_timeout=1): + """Constructor for S3FS objects. + + S3FS objects require the name of the S3 bucket in which to store + files, and can optionally be given a prefix under which the files + should be stored. The AWS public and private keys may be specified + as additional arguments; if they are not specified they will be + read from the two environment variables AWS_ACCESS_KEY_ID and + AWS_SECRET_ACCESS_KEY. + + The keyword argument 'key_sync_timeout' specifies the maximum + time in seconds that the filesystem will spend trying to confirm + that a newly-uploaded S3 key is available for reading. For no + timeout set it to zero. To disable these checks entirely (and + thus reduce the filesystem's consistency guarantees to those of + S3's "eventual consistency" model) set it to None. + + By default the path separator is '/', but this can be overridden + by specifying the keyword 'separator' in the constructor. + """ + self._bucket_name = bucket + self._access_keys = (aws_access_key, aws_secret_key) + self._separator = separator + self._key_sync_timeout = key_sync_timeout + # Normalise prefix to this form: path/to/files/ + prefix = normpath(prefix) + while prefix.startswith(separator): + prefix = prefix[1:] + if prefix and not prefix.endswith(separator): + prefix = prefix + separator + self._prefix = prefix + self._tlocal = thread_local() + super(S3FS, self).__init__(thread_synchronize=thread_synchronize) + + __repr__ = autorepr(['_bucket_name', '_access_keys[0]']) + + def _s3path(self, path): + """Get the absolute path to a file stored in S3.""" + path = relpath(normpath(path)) + path = self._separator.join(iteratepath(path)) + s3path = self._prefix + path + if s3path and s3path[-1] == self._separator: + s3path = s3path[:-1] + return s3path diff --git a/src/efs/filesystem.py b/src/efs/filesystem.py new file mode 100644 index 0000000..5fed298 --- /dev/null +++ b/src/efs/filesystem.py @@ -0,0 +1,122 @@ +"""The file system abstraction.""" +import urllib.parse + +from autorepr import autorepr +from flask import current_app + +from .efosfs import EFOSFS +from .efs3 import EFS3 + + +class EFS: + """The Eatfirst FileSystem.""" + + def __init__(self, storage='local', *args, **kwargs): + """The constructor method of the filesystem abstraction.""" + self.separator = kwargs.get('separator', '/') + self.current_file = '' + self.storage = storage + if storage.lower() == 'local': + self.home = EFOSFS(current_app.config['LOCAL_STORAGE'], create=True, *args, **kwargs) + elif storage.lower() == 's3': + self.home = EFS3(current_app.config['S3_BUCKET'], aws_access_key=current_app.config['AWS_ACCESS_KEY'], + aws_secret_key=current_app.config['AWS_SECRET_KEY'], *args, **kwargs) + else: + raise RuntimeError('{} does not support {} storage'.format(self.__class__.__name__, storage)) + + __repr__ = autorepr(['storage', 'separator', 'home']) + + def make_public(self, path): + """Make sure a file is public.""" + if hasattr(self.home, 'makepublic'): + self.home.makepublic(path) + + def upload(self, path, content, async_=False, content_type=None, *args, **kwargs): + """Upload a file and return its size in bytes. + + :param path: the relative path to file, including filename. + :param content: the content to be written. + :param async_: Create a thread to send the data in case True is sent. + :param content_type: Enforce content-type on destination. + :return: size of the saved file. + """ + path_list = path.split(self.separator) + if len(path_list) > 1: + self.home.makedir(self.separator.join(path_list[:-1]), recursive=True, allow_recreate=True) + self.home.createfile(path, wipe=False) + + if async_: + self.home.setcontents_async(path, content, *args, **kwargs).wait() + else: + self.home.setcontents(path, content, *args, **kwargs) + + if isinstance(self.home, EFS3) and content_type is not None: + # AWS is guessing the content type wrong. Bellow is our dirty fix for that. + key = self.home._s3bukt.get_key(path) + key.copy(key.bucket, key.name, preserve_acl=True, metadata={'Content-Type': content_type}) + + self.make_public(path) + + def open(self, path, *args, **kwargs): + """Open a file and return a file pointer. + + :param path: the relative path to file, including filename. + :return: a pointer to the file. + """ + if not self.home.exists(path): + exp = FileNotFoundError() + exp.filename = path + raise exp + return self.home.safeopen(path, *args, **kwargs) + + def remove(self, path): + """Remove a file or folder. + + :param path: the relative path to file, including filename. + """ + if self.home.isdir(path): + self.home.removedir(path, force=True) + else: + self.home.remove(path) + + def rename(self, path, new_path): + """Rename a file. + + :param path: the relative path to file, including filename. + :param path: the relative path to new file, including new filename. + """ + self.home.rename(path, new_path) + + def move(self, path, new_path): + """Move a file. + + :param path: the relative path to file, including filename. + :param new_path: the relative path to new file, including filename. + """ + path_list = new_path.split(self.separator) + if len(path_list) > 1: + self.home.makedir(self.separator.join(path_list[:-1]), recursive=True, allow_recreate=True) + self.home.move(path, new_path, overwrite=True) + + def file_url(self, path, with_cdn=True): + """Get a file url. + + :param path: the relative path to file, including filename. + :param with_cdn: specify if the url should return with the cdn information, only used for images. + """ + if not self.home.haspathurl(path): + if isinstance(self.home, EFOSFS): + return path + raise PermissionError('The file {} has no defined url'.format(path)) + + url = self.home.getpathurl(path) + if current_app.config.get('S3_CDN_URL', None) and with_cdn: + parsed_url = urllib.parse.urlparse(url) + url = url.replace(parsed_url.hostname, current_app.config['S3_CDN_URL']) + url = url.replace('http://', 'https://') + return url + + @classmethod + def get_filesystem(cls): + """Return an instance of the filesystem abstraction.""" + return cls(storage=current_app.config['DEFAULT_STORAGE']) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5a6a022 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,61 @@ +import os +import shutil +import tempfile + +import boto +import pytest +from flask import Flask + + +@pytest.yield_fixture(scope='function') +def app(): + """App fixture.""" + app_ = Flask(__name__) + app_.config['STORAGE'] = 's3' + + context = app_.app_context() + context.push() + + yield app_ + context.pop() + + +@pytest.fixture(scope='function') +def delete_temp_files(app, request): + """Remove files created by filesystem tests.""" + app.config['LOCAL_STORAGE'] = tempfile.mkdtemp() + + def teardown(): + """Define the teardown function.""" + home_path = app.config['LOCAL_STORAGE'] + if os.path.exists(home_path): + shutil.rmtree(home_path) + request.addfinalizer(teardown) + return True + + +@pytest.fixture(scope='function') +def bucket(app): + """A fixture to inject a function to create buckets. + + It has to return a function because as pytest setup fixtures before the function is called there is no time to + mock s3.""" + app.config['AWS_ACCESS_KEY'] = 'access-key' + app.config['AWS_SECRET_KEY'] = 'secret-key' + app.config['S3_BUCKET'] = 'bucket' + + def create_connection(): + """Define connection to get the bucket. + + This S3_BUCKET bucket should be pre-created manually before the tests. + """ + conn = boto.connect_s3(app.config['AWS_ACCESS_KEY'], app.config['AWS_SECRET_KEY']) + + # The virtual bucket needs to be created, see https://github.com/spulec/moto. + conn.create_bucket(app.config['S3_BUCKET']) + + bucket_ = conn.get_bucket(app.config['S3_BUCKET']) + + return bucket_ + + return create_connection diff --git a/tests/test_efs.py b/tests/test_efs.py deleted file mode 100644 index f88e0b6..0000000 --- a/tests/test_efs.py +++ /dev/null @@ -1,6 +0,0 @@ - -import efs - - -def test_main(): - assert efs # use your library here diff --git a/tests/test_local_filesystem.py b/tests/test_local_filesystem.py new file mode 100644 index 0000000..7f5329a --- /dev/null +++ b/tests/test_local_filesystem.py @@ -0,0 +1,94 @@ +"""Eatfirst FileSystem tests.""" + +import os +from io import StringIO + +from faker import Faker +from flask import current_app + +from efs import EFS + +fake = Faker() +TEST_FILE = 'test_file.txt' +RANDOM_DATA = StringIO(fake.sha256()) + + +def test_create_file(app, delete_temp_files): + """Test file creation.""" + assert app + assert delete_temp_files + home_path = current_app.config['LOCAL_STORAGE'] + efs = EFS() + efs.upload(TEST_FILE, RANDOM_DATA) + + assert os.path.exists(os.path.join(home_path, TEST_FILE)) + + +def test_read_file(app, delete_temp_files): + """Test read file.""" + assert app + assert delete_temp_files + efs = EFS() + efs.upload(TEST_FILE, RANDOM_DATA) + + uploaded_content = efs.open(TEST_FILE) + content = uploaded_content.read() + assert content == RANDOM_DATA.read() + + +def test_remove_file(app, delete_temp_files): + """Test remove file.""" + assert app + assert delete_temp_files + + efs = EFS() + home_path = current_app.config['LOCAL_STORAGE'] + efs.upload(TEST_FILE, RANDOM_DATA) + assert os.path.exists(os.path.join(home_path, TEST_FILE)) + + efs.remove(TEST_FILE) + assert not os.path.exists(os.path.join(home_path, TEST_FILE)) + + +def test_remove_folder(app, delete_temp_files): + """Test remove folder with content.""" + assert app + assert delete_temp_files + + efs = EFS() + home_path = current_app.config['LOCAL_STORAGE'] + efs.upload('very/long/path/to/be/created/' + TEST_FILE, RANDOM_DATA) + assert os.path.exists(os.path.join(home_path, 'very/long/path/to/be/created/', TEST_FILE)) + + efs.remove('very') + assert not os.path.exists(os.path.join(home_path, 'very')) + + +def test_rename_file(app, delete_temp_files): + """Test rename file.""" + assert app + assert delete_temp_files + + efs = EFS() + home_path = current_app.config['LOCAL_STORAGE'] + efs.upload(TEST_FILE, RANDOM_DATA) + assert os.path.exists(os.path.join(home_path, TEST_FILE)) + + efs.rename(TEST_FILE, 'new_test_file.txt') + assert not os.path.exists(os.path.join(home_path, TEST_FILE)) + assert os.path.exists(os.path.join(home_path, 'new_test_file.txt')) + + +def test_move_file(app, delete_temp_files): + """Test move file to a new folder.""" + assert app + assert delete_temp_files + + efs = EFS() + home_path = current_app.config['LOCAL_STORAGE'] + efs.upload(TEST_FILE, RANDOM_DATA) + assert os.path.exists(os.path.join(home_path, TEST_FILE)) + + efs.move(TEST_FILE, 'special_text/test_file.txt') + assert not os.path.exists(os.path.join(home_path, TEST_FILE)) + assert os.path.exists(os.path.join(home_path, 'special_text/' + TEST_FILE)) diff --git a/tests/test_s3_filesystem.py b/tests/test_s3_filesystem.py new file mode 100644 index 0000000..73d29c7 --- /dev/null +++ b/tests/test_s3_filesystem.py @@ -0,0 +1,111 @@ +"""Eatfirst FileSystem tests.""" +from io import StringIO +from uuid import uuid4 + +from faker import Faker +from moto import mock_s3_deprecated as mock_s3 + +from efs import EFS + +fake = Faker() +TEST_FILE = 'test_file_{0}.txt'.format(uuid4()) +RANDOM_DATA = StringIO(fake.sha256()) + + +@mock_s3 +def test_create_file(bucket): + """Test file creation.""" + bucket = bucket() + + efs = EFS(storage='s3') + RANDOM_DATA.seek(0) + efs.upload(TEST_FILE, RANDOM_DATA, async_=False) + + key = bucket.get_key(TEST_FILE) + RANDOM_DATA.seek(0) + assert key.get_contents_as_string().decode() == RANDOM_DATA.read() + + +@mock_s3 +def test_read_file(bucket): + """Test read file.""" + bucket = bucket() + k = bucket.new_key(TEST_FILE) + RANDOM_DATA.seek(0) + k.set_contents_from_file(RANDOM_DATA) + RANDOM_DATA.seek(0) + + efs = EFS(storage='s3') + uploaded_content = efs.open(TEST_FILE) + content = uploaded_content.read() + assert content == RANDOM_DATA.read() + + +@mock_s3 +def test_remove_file(bucket): + """Test remove file.""" + bucket = bucket() + + assert len(bucket.get_all_keys()) == 0 + k = bucket.new_key(TEST_FILE) + RANDOM_DATA.seek(0) + k.set_contents_from_file(RANDOM_DATA) + RANDOM_DATA.seek(0) + assert len(bucket.get_all_keys()) == 1 + + efs = EFS(storage='s3') + efs.remove(TEST_FILE) + assert len(bucket.get_all_keys()) == 0 + + +@mock_s3 +def test_upload_and_remove_folder(bucket): + """Test remove folder with content.""" + bucket = bucket() + + efs = EFS(storage='s3') + RANDOM_DATA.seek(0) + efs.upload('very/long/path/to/be/created/' + TEST_FILE, RANDOM_DATA, async_=False) + key = bucket.get_key('very/long/path/to/be/created/' + TEST_FILE) + assert key + + efs.remove('very/') + key = bucket.get_key('very/long/path/to/be/created/' + TEST_FILE) + assert not key + + +@mock_s3 +def test_rename_file(bucket): + """Test rename file.""" + bucket = bucket() + + efs = EFS(storage='s3') + RANDOM_DATA.seek(0) + efs.upload(TEST_FILE, RANDOM_DATA, async_=False) + key = bucket.get_key(TEST_FILE) + assert key + + efs.rename(TEST_FILE, 'new_test_file.txt') + key = bucket.get_key(TEST_FILE) + assert not key + key = bucket.get_key('new_test_file.txt') + assert key + + +@mock_s3 +def test_move_file(bucket): + """Test move file to a new folder.""" + bucket = bucket() + + efs = EFS(storage='s3') + RANDOM_DATA.seek(0) + efs.upload(TEST_FILE, RANDOM_DATA, async_=False) + + key = bucket.get_key(TEST_FILE) + assert key + + efs.move(TEST_FILE, 'special_text/test_file.txt') + key = bucket.get_key(TEST_FILE) + assert not key + key = bucket.get_key('special_text/test_file.txt') + assert key diff --git a/tox.ini b/tox.ini index 508a01a..bf1ef5e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,19 +4,16 @@ envlist = clean, check, - {py27,py33,py34,py35,py36,pypy}, + {py34,py35,py36}, report, docs [testenv] basepython = - pypy: {env:TOXPYTHON:pypy} - {py27,docs,spell}: {env:TOXPYTHON:python2.7} - py33: {env:TOXPYTHON:python3.3} py34: {env:TOXPYTHON:python3.4} py35: {env:TOXPYTHON:python3.5} py36: {env:TOXPYTHON:python3.6} - {bootstrap,clean,check,report,coveralls,codecov}: {env:TOXPYTHON:python3} + {bootstrap,clean,check,docs,report,coveralls,codecov}: {env:TOXPYTHON:python3} setenv = PYTHONPATH={toxinidir}/tests PYTHONUNBUFFERED=yes @@ -24,6 +21,8 @@ passenv = * usedevelop = false deps = + faker + moto pytest pytest-travis-fold pytest-cov From 841fe62b324e1a3699360387ea2f6a2e830a4ac8 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Mon, 16 Oct 2017 10:09:28 +0200 Subject: [PATCH 2/3] Changes requested on code review Signed-off-by: Jayson Reis --- docs/usage.rst | 10 ++++++++-- src/efs/{efosfs.py => eatfirst_osfs.py} | 2 +- src/efs/{efs3.py => eatfirst_s3.py} | 6 +++--- src/efs/filesystem.py | 16 ++++++++-------- tests/test_local_filesystem.py | 2 +- tests/test_s3_filesystem.py | 2 +- 6 files changed, 22 insertions(+), 16 deletions(-) rename src/efs/{efosfs.py => eatfirst_osfs.py} (85%) rename src/efs/{efs3.py => eatfirst_s3.py} (93%) diff --git a/docs/usage.rst b/docs/usage.rst index be6eba7..ed7f0e5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2,6 +2,12 @@ Usage ===== -To use EatFirst fs + flask wrapper in a project:: +To use EatFirst fs + flask wrapper in a project: - import efs +.. code-block:: python + + import efs + + # There is no need to initialise it because it will always the current app. + fs = efs.get_filesystem() + fs.upload('a/file.txt', open('/tmp/file.txt', 'rb')) diff --git a/src/efs/efosfs.py b/src/efs/eatfirst_osfs.py similarity index 85% rename from src/efs/efosfs.py rename to src/efs/eatfirst_osfs.py index 891418d..f8ef6ff 100644 --- a/src/efs/efosfs.py +++ b/src/efs/eatfirst_osfs.py @@ -2,7 +2,7 @@ from fs.osfs import OSFS -class EFOSFS(OSFS): +class EatFirstOSFS(OSFS): """Simple wrapper to have a better repr.""" __repr__ = autorepr(['root_path', 'dir_mode:o']) diff --git a/src/efs/efs3.py b/src/efs/eatfirst_s3.py similarity index 93% rename from src/efs/efs3.py rename to src/efs/eatfirst_s3.py index e24d299..b5ab943 100644 --- a/src/efs/efs3.py +++ b/src/efs/eatfirst_s3.py @@ -1,4 +1,4 @@ -"""Eatfirst FileSystem for S3.""" +"""EatFirst file system for S3.""" from autorepr import autorepr from fs.path import iteratepath @@ -8,7 +8,7 @@ from fs.s3fs import thread_local -class EFS3(S3FS): +class EatFirstS3(S3FS): """Extension of S3FS class, fixing it to work with python 3k.""" def __init__(self, bucket, prefix='', aws_access_key=None, aws_secret_key=None, separator='/', @@ -46,7 +46,7 @@ def __init__(self, bucket, prefix='', aws_access_key=None, aws_secret_key=None, self._tlocal = thread_local() super(S3FS, self).__init__(thread_synchronize=thread_synchronize) - __repr__ = autorepr(['_bucket_name', '_access_keys[0]']) + __repr__ = autorepr([], bucket=lambda self: self._bucket_name, key=lambda self: self._access_keys[0]) def _s3path(self, path): """Get the absolute path to a file stored in S3.""" diff --git a/src/efs/filesystem.py b/src/efs/filesystem.py index 5fed298..9e678f7 100644 --- a/src/efs/filesystem.py +++ b/src/efs/filesystem.py @@ -4,12 +4,12 @@ from autorepr import autorepr from flask import current_app -from .efosfs import EFOSFS -from .efs3 import EFS3 +from .eatfirst_osfs import EatFirstOSFS +from .eatfirst_s3 import EatFirstS3 class EFS: - """The Eatfirst FileSystem.""" + """The EatFirst File system.""" def __init__(self, storage='local', *args, **kwargs): """The constructor method of the filesystem abstraction.""" @@ -17,10 +17,10 @@ def __init__(self, storage='local', *args, **kwargs): self.current_file = '' self.storage = storage if storage.lower() == 'local': - self.home = EFOSFS(current_app.config['LOCAL_STORAGE'], create=True, *args, **kwargs) + self.home = EatFirstOSFS(current_app.config['LOCAL_STORAGE'], create=True, *args, **kwargs) elif storage.lower() == 's3': - self.home = EFS3(current_app.config['S3_BUCKET'], aws_access_key=current_app.config['AWS_ACCESS_KEY'], - aws_secret_key=current_app.config['AWS_SECRET_KEY'], *args, **kwargs) + self.home = EatFirstS3(current_app.config['S3_BUCKET'], aws_access_key=current_app.config['AWS_ACCESS_KEY'], + aws_secret_key=current_app.config['AWS_SECRET_KEY'], *args, **kwargs) else: raise RuntimeError('{} does not support {} storage'.format(self.__class__.__name__, storage)) @@ -50,7 +50,7 @@ def upload(self, path, content, async_=False, content_type=None, *args, **kwargs else: self.home.setcontents(path, content, *args, **kwargs) - if isinstance(self.home, EFS3) and content_type is not None: + if isinstance(self.home, EatFirstS3) and content_type is not None: # AWS is guessing the content type wrong. Bellow is our dirty fix for that. key = self.home._s3bukt.get_key(path) key.copy(key.bucket, key.name, preserve_acl=True, metadata={'Content-Type': content_type}) @@ -105,7 +105,7 @@ def file_url(self, path, with_cdn=True): :param with_cdn: specify if the url should return with the cdn information, only used for images. """ if not self.home.haspathurl(path): - if isinstance(self.home, EFOSFS): + if isinstance(self.home, EatFirstOSFS): return path raise PermissionError('The file {} has no defined url'.format(path)) diff --git a/tests/test_local_filesystem.py b/tests/test_local_filesystem.py index 7f5329a..b4b0de7 100644 --- a/tests/test_local_filesystem.py +++ b/tests/test_local_filesystem.py @@ -1,4 +1,4 @@ -"""Eatfirst FileSystem tests.""" +"""EatFirst FileSystem tests.""" import os from io import StringIO diff --git a/tests/test_s3_filesystem.py b/tests/test_s3_filesystem.py index 73d29c7..9c845e1 100644 --- a/tests/test_s3_filesystem.py +++ b/tests/test_s3_filesystem.py @@ -1,4 +1,4 @@ -"""Eatfirst FileSystem tests.""" +"""EatFirst FileSystem tests.""" from io import StringIO from uuid import uuid4 From feb775497d676a4f5d90d537ce34199a319752cd Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Mon, 16 Oct 2017 10:45:23 +0200 Subject: [PATCH 3/3] Fix typo Signed-off-by: Jayson Reis --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index ed7f0e5..a8d264c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -8,6 +8,6 @@ To use EatFirst fs + flask wrapper in a project: import efs - # There is no need to initialise it because it will always the current app. + # There is no need to initialise it because it will always use the current app. fs = efs.get_filesystem() fs.upload('a/file.txt', open('/tmp/file.txt', 'rb'))