eatfirst · insanoid · Oct 16, 2017 · Oct 13, 2017 · Oct 16, 2017 · Oct 16, 2017
diff --git a/.editorconfig b/.editorconfig
@@ -11,3 +11,6 @@ charset = utf-8
 
 [*.{bat,cmd,ps1}]
 end_of_line = crlf
+
+[*.{yml}]
+indent_size = 2
diff --git a/.travis.yml b/.travis.yml
@@ -10,12 +10,6 @@ env:
     - TOXENV=docs
 matrix:
   include:
-    - python: '2.7'
-      env:
-        - TOXENV=py27,report
-    - python: '3.3'
-      env:
-        - TOXENV=py33,report
     - python: '3.4'
       env:
         - TOXENV=py34,report
@@ -25,9 +19,6 @@ matrix:
     - python: '3.6'
       env:
         - TOXENV=py36,report
-    - python: 'pypy-5.4'
-      env:
-        - TOXENV=pypy,report
 before_install:
   - python --version
   - uname -a

diff --git a/appveyor.yml b/appveyor.yml
@@ -7,32 +7,10 @@ environment:
     WITH_COMPILER: 'cmd /E:ON /V:ON /C .\ci\appveyor-with-compiler.cmd'
   matrix:
     - TOXENV: check
-      TOXPYTHON: C:\Python27\python.exe
-      PYTHON_HOME: C:\Python27
-      PYTHON_VERSION: '2.7'
-      PYTHON_ARCH: '32'
-    - TOXENV: 'py27,report'
-      TOXPYTHON: C:\Python27\python.exe
-      PYTHON_HOME: C:\Python27
-      PYTHON_VERSION: '2.7'
-      PYTHON_ARCH: '32'
-    - TOXENV: 'py27,report'
-      TOXPYTHON: C:\Python27-x64\python.exe
-      WINDOWS_SDK_VERSION: v7.0
-      PYTHON_HOME: C:\Python27-x64
-      PYTHON_VERSION: '2.7'
-      PYTHON_ARCH: '64'
-    - TOXENV: 'py33,report'
-      TOXPYTHON: C:\Python33\python.exe
-      PYTHON_HOME: C:\Python33
-      PYTHON_VERSION: '3.3'
+      TOXPYTHON: C:\Python34\python.exe
+      PYTHON_HOME: C:\Python34
+      PYTHON_VERSION: '3.4'
       PYTHON_ARCH: '32'
-    - TOXENV: 'py33,report'
-      TOXPYTHON: C:\Python33-x64\python.exe
-      WINDOWS_SDK_VERSION: v7.1
-      PYTHON_HOME: C:\Python33-x64
-      PYTHON_VERSION: '3.3'
-      PYTHON_ARCH: '64'
     - TOXENV: 'py34,report'
       TOXPYTHON: C:\Python34\python.exe
       PYTHON_HOME: C:\Python34

diff --git a/ci/templates/appveyor.yml b/ci/templates/appveyor.yml
@@ -7,9 +7,9 @@ environment:
     WITH_COMPILER: 'cmd /E:ON /V:ON /C .\ci\appveyor-with-compiler.cmd'
   matrix:
     - TOXENV: check
-      TOXPYTHON: C:\Python27\python.exe
-      PYTHON_HOME: C:\Python27
-      PYTHON_VERSION: '2.7'
+      TOXPYTHON: C:\Python34\python.exe
+      PYTHON_HOME: C:\Python34
+      PYTHON_VERSION: '3.4'
       PYTHON_ARCH: '32'
 {% for env in tox_environments %}{{ '' }}{% if env.startswith(('py2', 'py3')) %}
     - TOXENV: '{{ env }},report'

diff --git a/docs/usage.rst b/docs/usage.rst
@@ -2,6 +2,12 @@
 Usage
 =====
 
-To use EatFirst fs + flask wrapper in a project::
+To use EatFirst fs + flask wrapper in a project:
 
-	import efs
+.. code-block:: python
+
+    import efs
+
+    # There is no need to initialise it because it will always use the current app.
+    fs = efs.get_filesystem()
+    fs.upload('a/file.txt', open('/tmp/file.txt', 'rb'))
diff --git a/setup.py b/setup.py
@@ -56,17 +56,16 @@ def read(*names, **kwargs):
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
-        # uncomment if you test on these interpreters:
-        # 'Programming Language :: Python :: Implementation :: IronPython',
-        # 'Programming Language :: Python :: Implementation :: Jython',
-        # 'Programming Language :: Python :: Implementation :: Stackless',
         'Topic :: Utilities',
     ],
     keywords=[
         # eg: 'keyword1', 'keyword2', 'keyword3',
     ],
     install_requires=[
-        # eg: 'aspectlib==1.1.1', 'six>=1.7',
+        'Flask',
+        'fs >= 0.5.4, < 2.0',
+        'autorepr',
+        'boto'
     ],
     extras_require={
         # eg:

diff --git a/src/efs/__init__.py b/src/efs/__init__.py
@@ -1 +1,5 @@
 __version__ = "0.1.0"
+
+from .filesystem import EFS
+
+__all__ = ('EFS', )
diff --git a/src/efs/eatfirst_osfs.py b/src/efs/eatfirst_osfs.py
@@ -0,0 +1,8 @@
+from autorepr import autorepr
+from fs.osfs import OSFS
+
+
+class EatFirstOSFS(OSFS):
+    """Simple wrapper to have a better repr."""
+
+    __repr__ = autorepr(['root_path', 'dir_mode:o'])
diff --git a/src/efs/eatfirst_s3.py b/src/efs/eatfirst_s3.py
@@ -0,0 +1,58 @@
+"""EatFirst file system for S3."""
+
+from autorepr import autorepr
+from fs.path import iteratepath
+from fs.path import normpath
+from fs.path import relpath
+from fs.s3fs import S3FS
+from fs.s3fs import thread_local
+
+
+class EatFirstS3(S3FS):
+    """Extension of S3FS class, fixing it to work with python 3k."""
+
+    def __init__(self, bucket, prefix='', aws_access_key=None, aws_secret_key=None, separator='/',
+                 thread_synchronize=True, key_sync_timeout=1):
+        """Constructor for S3FS objects.
+
+        S3FS objects require the name of the S3 bucket in which to store
+        files, and can optionally be given a prefix under which the files
+        should be stored.  The AWS public and private keys may be specified
+        as additional arguments; if they are not specified they will be
+        read from the two environment variables AWS_ACCESS_KEY_ID and
+        AWS_SECRET_ACCESS_KEY.
+
+        The keyword argument 'key_sync_timeout' specifies the maximum
+        time in seconds that the filesystem will spend trying to confirm
+        that a newly-uploaded S3 key is available for reading.  For no
+        timeout set it to zero.  To disable these checks entirely (and
+        thus reduce the filesystem's consistency guarantees to those of
+        S3's "eventual consistency" model) set it to None.
+
+        By default the path separator is '/', but this can be overridden
+        by specifying the keyword 'separator' in the constructor.
+        """
+        self._bucket_name = bucket
+        self._access_keys = (aws_access_key, aws_secret_key)
+        self._separator = separator
+        self._key_sync_timeout = key_sync_timeout
+        # Normalise prefix to this form: path/to/files/
+        prefix = normpath(prefix)
+        while prefix.startswith(separator):
+            prefix = prefix[1:]
+        if prefix and not prefix.endswith(separator):
+            prefix = prefix + separator
+        self._prefix = prefix
+        self._tlocal = thread_local()
+        super(S3FS, self).__init__(thread_synchronize=thread_synchronize)
+
+    __repr__ = autorepr([], bucket=lambda self: self._bucket_name, key=lambda self: self._access_keys[0])
+
+    def _s3path(self, path):
+        """Get the absolute path to a file stored in S3."""
+        path = relpath(normpath(path))
+        path = self._separator.join(iteratepath(path))
+        s3path = self._prefix + path
+        if s3path and s3path[-1] == self._separator:
+            s3path = s3path[:-1]
+        return s3path
diff --git a/src/efs/filesystem.py b/src/efs/filesystem.py
@@ -0,0 +1,122 @@
+"""The file system abstraction."""
+import urllib.parse
+
+from autorepr import autorepr
+from flask import current_app
+
+from .eatfirst_osfs import EatFirstOSFS
+from .eatfirst_s3 import EatFirstS3
+
+
+class EFS:
+    """The EatFirst File system."""
+
+    def __init__(self, storage='local', *args, **kwargs):
+        """The constructor method of the filesystem abstraction."""
+        self.separator = kwargs.get('separator', '/')
+        self.current_file = ''
+        self.storage = storage
+        if storage.lower() == 'local':
+            self.home = EatFirstOSFS(current_app.config['LOCAL_STORAGE'], create=True, *args, **kwargs)
+        elif storage.lower() == 's3':
+            self.home = EatFirstS3(current_app.config['S3_BUCKET'], aws_access_key=current_app.config['AWS_ACCESS_KEY'],
+                                   aws_secret_key=current_app.config['AWS_SECRET_KEY'], *args, **kwargs)
+        else:
+            raise RuntimeError('{} does not support {} storage'.format(self.__class__.__name__, storage))
+
+    __repr__ = autorepr(['storage', 'separator', 'home'])
+
+    def make_public(self, path):
+        """Make sure a file is public."""
+        if hasattr(self.home, 'makepublic'):
+            self.home.makepublic(path)
+
+    def upload(self, path, content, async_=False, content_type=None, *args, **kwargs):
+        """Upload a file and return its size in bytes.
+
+        :param path: the relative path to file, including filename.
+        :param content: the content to be written.
+        :param async_: Create a thread to send the data in case True is sent.
+        :param content_type: Enforce content-type on destination.
+        :return: size of the saved file.
+        """
+        path_list = path.split(self.separator)
+        if len(path_list) > 1:
+            self.home.makedir(self.separator.join(path_list[:-1]), recursive=True, allow_recreate=True)
+        self.home.createfile(path, wipe=False)
+
+        if async_:
+            self.home.setcontents_async(path, content, *args, **kwargs).wait()
+        else:
+            self.home.setcontents(path, content, *args, **kwargs)
+
+        if isinstance(self.home, EatFirstS3) and content_type is not None:
+            # AWS is guessing the content type wrong. Bellow is our dirty fix for that.
+            key = self.home._s3bukt.get_key(path)
+            key.copy(key.bucket, key.name, preserve_acl=True, metadata={'Content-Type': content_type})
+
+        self.make_public(path)
+
+    def open(self, path, *args, **kwargs):
+        """Open a file and return a file pointer.
+
+        :param path: the relative path to file, including filename.
+        :return: a pointer to the file.
+        """
+        if not self.home.exists(path):
+            exp = FileNotFoundError()
+            exp.filename = path
+            raise exp
+        return self.home.safeopen(path, *args, **kwargs)
+
+    def remove(self, path):
+        """Remove a file or folder.
+
+        :param path: the relative path to file, including filename.
+        """
+        if self.home.isdir(path):
+            self.home.removedir(path, force=True)
+        else:
+            self.home.remove(path)
+
+    def rename(self, path, new_path):
+        """Rename a file.
+
+        :param path: the relative path to file, including filename.
+        :param path: the relative path to new file, including new filename.
+        """
+        self.home.rename(path, new_path)
+
+    def move(self, path, new_path):
+        """Move a file.
+
+        :param path: the relative path to file, including filename.
+        :param new_path: the relative path to new file, including filename.
+        """
+        path_list = new_path.split(self.separator)
+        if len(path_list) > 1:
+            self.home.makedir(self.separator.join(path_list[:-1]), recursive=True, allow_recreate=True)
+        self.home.move(path, new_path, overwrite=True)
+
+    def file_url(self, path, with_cdn=True):
+        """Get a file url.
+
+        :param path: the relative path to file, including filename.
+        :param with_cdn: specify if the url should return with the cdn information, only used for images.
+        """
+        if not self.home.haspathurl(path):
+            if isinstance(self.home, EatFirstOSFS):
+                return path
+            raise PermissionError('The file {} has no defined url'.format(path))
+
+        url = self.home.getpathurl(path)
+        if current_app.config.get('S3_CDN_URL', None) and with_cdn:
+            parsed_url = urllib.parse.urlparse(url)
+            url = url.replace(parsed_url.hostname, current_app.config['S3_CDN_URL'])
+        url = url.replace('http://', 'https://')
+        return url
+
+    @classmethod
+    def get_filesystem(cls):
+        """Return an instance of the filesystem abstraction."""
+        return cls(storage=current_app.config['DEFAULT_STORAGE'])
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,61 @@
+import os
+import shutil
+import tempfile
+
+import boto
+import pytest
+from flask import Flask
+
+
+@pytest.yield_fixture(scope='function')
+def app():
+    """App fixture."""
+    app_ = Flask(__name__)
+    app_.config['STORAGE'] = 's3'
+
+    context = app_.app_context()
+    context.push()
+
+    yield app_
+    context.pop()
+
+
+@pytest.fixture(scope='function')
+def delete_temp_files(app, request):
+    """Remove files created by filesystem tests."""
+    app.config['LOCAL_STORAGE'] = tempfile.mkdtemp()
+
+    def teardown():
+        """Define the teardown function."""
+        home_path = app.config['LOCAL_STORAGE']
+        if os.path.exists(home_path):
+            shutil.rmtree(home_path)
+    request.addfinalizer(teardown)
+    return True
+
+
+@pytest.fixture(scope='function')
+def bucket(app):
+    """A fixture to inject a function to create buckets.
+
+    It has to return a function because as pytest setup fixtures before the function is called there is no time to
+    mock s3."""
+    app.config['AWS_ACCESS_KEY'] = 'access-key'
+    app.config['AWS_SECRET_KEY'] = 'secret-key'
+    app.config['S3_BUCKET'] = 'bucket'
+
+    def create_connection():
+        """Define connection to get the bucket.
+
+        This S3_BUCKET bucket should be pre-created manually before the tests.
+        """
+        conn = boto.connect_s3(app.config['AWS_ACCESS_KEY'], app.config['AWS_SECRET_KEY'])
+
+        # The virtual bucket needs to be created, see https://github.com/spulec/moto.
+        conn.create_bucket(app.config['S3_BUCKET'])
+
+        bucket_ = conn.get_bucket(app.config['S3_BUCKET'])
+
+        return bucket_
+
+    return create_connection
diff --git a/tests/test_efs.py b/tests/test_efs.py