docarray · samsja · Mar 28, 2023 · Mar 28, 2023 · Mar 28, 2023 · Mar 28, 2023
diff --git a/docarray/array/abstract_array.py b/docarray/array/abstract_array.py
@@ -22,7 +22,7 @@
 from docarray.base_doc import BaseDoc
 from docarray.display.document_array_summary import DocArraySummary
 from docarray.typing.abstract_type import AbstractType
-from docarray.utils._typing import change_cls_name
+from docarray.utils._internal._typing import change_cls_name
 
 if TYPE_CHECKING:
     from docarray.proto import DocArrayProto, NodeProto

diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py
@@ -31,7 +31,7 @@
     _all_access_paths_valid,
     _dict_to_access_paths,
 )
-from docarray.utils.compress import _decompress_bytes, _get_compress_ctx
+from docarray.utils._internal.compress import _decompress_bytes, _get_compress_ctx
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -201,7 +201,7 @@ def to_binary_stream(
         from rich import filesize
 
         if show_progress:
-            from docarray.utils.progress_bar import _get_progressbar
+            from docarray.utils._internal.progress_bar import _get_progressbar
 
             pbar, t = _get_progressbar(
                 'Serializing', disable=not show_progress, total=len(self)
@@ -564,7 +564,7 @@ def _load_binary_all(
         else:
             from rich import filesize
 
-            from docarray.utils.progress_bar import _get_progressbar
+            from docarray.utils._internal.progress_bar import _get_progressbar
 
             # 1 byte (uint8)
             # 8 bytes (uint64)
@@ -629,7 +629,7 @@ def _load_binary_stream(
             num_docs = int.from_bytes(version_numdocs_lendoc0[1:9], 'big', signed=False)
 
             if show_progress:
-                from docarray.utils.progress_bar import _get_progressbar
+                from docarray.utils._internal.progress_bar import _get_progressbar
 
                 pbar, t = _get_progressbar(
                     'Deserializing', disable=not show_progress, total=num_docs

diff --git a/docarray/array/array/sequence_indexing_mixin.py b/docarray/array/array/sequence_indexing_mixin.py
@@ -14,7 +14,7 @@
 
 import numpy as np
 
-from docarray.utils.misc import is_torch_available
+from docarray.utils._internal.misc import is_torch_available
 
 T_item = TypeVar('T_item')
 T = TypeVar('T', bound='IndexingSequenceMixin')

diff --git a/docarray/array/stacked/array_stacked.py b/docarray/array/stacked/array_stacked.py
@@ -26,8 +26,8 @@
 from docarray.base_doc.mixins.io import _type_to_protobuf
 from docarray.typing import NdArray
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
-from docarray.utils._typing import is_tensor_union
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal._typing import is_tensor_union
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 if TYPE_CHECKING:
     from pydantic.fields import ModelField

diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py
@@ -20,8 +20,8 @@
 from docarray.base_doc.base_node import BaseNode
 from docarray.typing import NdArray
 from docarray.typing.proto_register import _PROTO_TYPE_NAME_TO_CLASS
-from docarray.utils.compress import _compress_bytes, _decompress_bytes
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.compress import _compress_bytes, _decompress_bytes
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 tf_available = is_tf_available()
 if tf_available:

diff --git a/docarray/data/torch_dataset.py b/docarray/data/torch_dataset.py
@@ -4,7 +4,7 @@
 
 from docarray import BaseDoc, DocArray, DocArrayStacked
 from docarray.typing import TorchTensor
-from docarray.utils._typing import change_cls_name
+from docarray.utils._internal._typing import change_cls_name
 
 T_doc = TypeVar('T_doc', bound=BaseDoc)
 

diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py
@@ -7,7 +7,7 @@
 from docarray.typing.bytes.audio_bytes import AudioBytes
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
 from docarray.typing.tensor.audio.audio_tensor import AudioTensor
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 torch_available = is_torch_available()
 if torch_available:

diff --git a/docarray/documents/image.py b/docarray/documents/image.py
@@ -6,7 +6,7 @@
 from docarray.typing import AnyEmbedding, ImageBytes, ImageUrl
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
 from docarray.typing.tensor.image.image_tensor import ImageTensor
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 T = TypeVar('T', bound='ImageDoc')
 

diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py
@@ -6,7 +6,7 @@
 from docarray.documents.point_cloud.points_and_colors import PointsAndColors
 from docarray.typing import AnyEmbedding, PointCloud3DUrl
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 torch_available = is_torch_available()
 if torch_available:

diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py
@@ -5,7 +5,7 @@
 from docarray.base_doc import BaseDoc
 from docarray.typing import AnyTensor
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 torch_available = is_torch_available()
 if torch_available:

diff --git a/docarray/documents/video.py b/docarray/documents/video.py
@@ -8,7 +8,7 @@
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
 from docarray.typing.tensor.video.video_tensor import VideoTensor
 from docarray.typing.url.video_url import VideoUrl
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 torch_available = is_torch_available()
 if torch_available:

diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py
@@ -26,9 +26,9 @@
 from docarray import BaseDoc, DocArray
 from docarray.array.abstract_array import AnyDocArray
 from docarray.typing import AnyTensor
-from docarray.utils._typing import unwrap_optional_type
+from docarray.utils._internal._typing import unwrap_optional_type
+from docarray.utils._internal.misc import is_tf_available, torch_imported
 from docarray.utils.find import FindResult, _FindResult
-from docarray.utils.misc import is_tf_available, torch_imported
 
 if TYPE_CHECKING:
     from pydantic.fields import ModelField

diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py
@@ -29,9 +29,9 @@
     _raise_not_supported,
 )
 from docarray.proto import DocumentProto
+from docarray.utils._internal.misc import is_np_int, is_tf_available, is_torch_available
 from docarray.utils.filter import filter_docs
 from docarray.utils.find import _FindResult
-from docarray.utils.misc import is_np_int, is_tf_available, is_torch_available
 
 TSchema = TypeVar('TSchema', bound=BaseDoc)
 T = TypeVar('T', bound='HnswDocumentIndex')

diff --git a/docarray/store/file.py b/docarray/store/file.py
@@ -7,7 +7,7 @@
 from docarray.store.abstract_doc_store import AbstractDocStore
 from docarray.store.exceptions import ConcurrentPushException
 from docarray.store.helpers import _from_binary_stream, _to_binary_stream
-from docarray.utils.cache import get_cache_path
+from docarray.utils._internal.cache import _get_cache_path
 
 if TYPE_CHECKING:
     from docarray import BaseDoc, DocArray
@@ -23,7 +23,7 @@ def _abs_filepath(name: str) -> Path:
         If it is a path, it is resolved to an absolute path.
         """
         if not (name.startswith('/') or name.startswith('~') or name.startswith('.')):
-            name = str(get_cache_path() / name)
+            name = str(_get_cache_path() / name)
         if name.startswith('~'):
             name = str(Path.home() / name[2:])
         return Path(name).resolve()

diff --git a/docarray/store/helpers.py b/docarray/store/helpers.py
@@ -6,7 +6,7 @@
 from rich import filesize
 from typing_extensions import TYPE_CHECKING, Protocol
 
-from docarray.utils.progress_bar import _get_progressbar
+from docarray.utils._internal.progress_bar import _get_progressbar
 
 if TYPE_CHECKING:
     from pathlib import Path

diff --git a/docarray/store/jac.py b/docarray/store/jac.py
@@ -24,7 +24,7 @@
     get_version_info,
     raise_req_error,
 )
-from docarray.utils.cache import get_cache_path
+from docarray.utils._internal.cache import _get_cache_path
 
 if TYPE_CHECKING:  # pragma: no cover
     import io
@@ -331,7 +331,7 @@ def pull_stream(
                 _BufferedCachingRequestReader, io.BufferedReader
             ] = _BufferedCachingRequestReader(r, tmp_cache_file)
 
-            cache_file = get_cache_path() / f'{save_name}.da'
+            cache_file = _get_cache_path() / f'{save_name}.da'
             if local_cache and cache_file.exists():
                 _cache_len = cache_file.stat().st_size
                 if _cache_len == int(r.headers['Content-length']):
@@ -354,7 +354,7 @@ def pull_stream(
 
             if local_cache:
                 if isinstance(_source, _BufferedCachingRequestReader):
-                    Path(get_cache_path()).mkdir(parents=True, exist_ok=True)
+                    Path(_get_cache_path()).mkdir(parents=True, exist_ok=True)
                     tmp_cache_file.rename(cache_file)
                 else:
                     _source.close()
diff --git a/docarray/store/s3.py b/docarray/store/s3.py
@@ -10,7 +10,7 @@
 
 from docarray.store.abstract_doc_store import AbstractDocStore
 from docarray.store.helpers import _from_binary_stream, _to_binary_stream
-from docarray.utils.cache import get_cache_path
+from docarray.utils._internal.cache import _get_cache_path
 
 if TYPE_CHECKING:  # pragma: no cover
     from docarray import BaseDoc, DocArray
@@ -212,7 +212,7 @@ def pull_stream(
         bucket, name = name.split('/', 1)
 
         save_name = name.replace('/', '_')
-        cache_path = get_cache_path() / f'{save_name}.da'
+        cache_path = _get_cache_path() / f'{save_name}.da'
 
         source = _BufferedCachingReader(
             open(f"s3://{bucket}/{name}.da", 'rb', compression='.gz'),

diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py
@@ -1,4 +1,4 @@
-from docarray.typing.bytes import ImageBytes
+from docarray.typing.bytes import AudioBytes, ImageBytes, VideoBytes
 from docarray.typing.id import ID
 from docarray.typing.tensor import ImageNdArray, ImageTensor
 from docarray.typing.tensor.audio import AudioNdArray
@@ -31,14 +31,16 @@
     'AnyUrl',
     'ID',
     'AnyTensor',
-    'TensorFlowTensor',
     'NdArrayEmbedding',
     'ImageBytes',
     'ImageTensor',
     'ImageNdArray',
+    'ImageBytes',
+    'VideoBytes',
+    'AudioBytes',
 ]
 
-from docarray.utils.misc import is_tf_available, is_torch_available
+from docarray.utils._internal.misc import is_tf_available, is_torch_available
 
 torch_available = is_torch_available()
 if torch_available:
@@ -59,7 +61,7 @@
 
 tf_available = is_tf_available()
 if tf_available:
-    from docarray.typing.tensor import TensorFlowTensor
+    from docarray.typing.tensor import TensorFlowTensor  # noqa: F401
     from docarray.typing.tensor.audio import AudioTensorFlowTensor  # noqa: F401
     from docarray.typing.tensor.embedding import TensorFlowEmbedding  # noqa: F401
     from docarray.typing.tensor.image import ImageTensorFlowTensor  # noqa: F401

diff --git a/docarray/typing/bytes/__init__.py b/docarray/typing/bytes/__init__.py
@@ -1,3 +1,5 @@
+from docarray.typing.bytes.audio_bytes import AudioBytes
 from docarray.typing.bytes.image_bytes import ImageBytes
+from docarray.typing.bytes.video_bytes import VideoBytes
 
-__all__ = ['ImageBytes']
+__all__ = ['ImageBytes', 'VideoBytes', 'AudioBytes']
diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py
@@ -46,32 +46,35 @@ def load(self) -> Tuple[np.ndarray, int]:
         """
         Load the Audio from the bytes into a numpy.ndarray Audio tensor
 
-        EXAMPLE USAGE
+        ---
 
-        .. code-block:: python
+        ```python
+        from typing import Optional
+        from docarray import BaseDoc
+        from docarray.typing import AudioUrl, NdArray, AudioBytes
 
-            from docarray import BaseDoc
-            import numpy as np
+        import numpy as np
 
-            from docarray.typing import AudioUrl
 
+        class MyAudio(BaseDoc):
+            url: AudioUrl
+            tensor: Optional[NdArray]
+            bytes: Optional[AudioBytes]
+            frame_rate: Optional[float]
 
-            class MyAudio(Document):
-                url: AudioUrl
-                tensor: Optional[NdArray]
-                bytes: Optional[bytes]
 
+        doc = MyAudio(url='https://www.kozco.com/tech/piano2.wav')
+        doc.bytes = doc.url.load_bytes()
+        doc.tensor, doc.frame_rate = doc.bytes.load()
 
-            doc = MyAudio(url="toydata/hello.wav")
-            doc.bytes = doc.url.load_bytes()
-            doc.tensor, doc.frame_rate = doc.bytes.load()
+        # Note this is equivalent to do
 
-            # Note this is equivalent to do
+        doc.tensor, doc.frame_rate = doc.url.load()
 
-            doc.tensor, doc.frame_rate = doc.url.load()
-
-            assert isinstance(doc.audio_tensor, np.ndarray)
+        assert isinstance(doc.tensor, np.ndarray)
+        ```
 
+        ---
         :return: np.ndarray representing the Audio as RGB values
         """
 

diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py
@@ -51,34 +51,35 @@ def load(
         """
         Load the image from the bytes into a numpy.ndarray image tensor
 
-        EXAMPLE USAGE
+        ---
 
-        .. code-block:: python
+        ```python
+        from docarray import BaseDoc
+        from docarray.typing import ImageUrl
+        import numpy as np
 
-            from docarray import BaseDoc
-            from docarray.typing import ImageUrl
-            import numpy as np
 
+        class MyDoc(BaseDoc):
+            img_url: ImageUrl
 
-            class MyDoc(BaseDoc):
-                img_url: ImageUrl
 
+        doc = MyDoc(
+            img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/"
+            "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg"
+        )
 
-            doc = MyDoc(
-                img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/"
-                "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg"
-            )
+        img_tensor = doc.img_url.load()
+        assert isinstance(img_tensor, np.ndarray)
 
-            img_tensor = doc.img_url.load()
-            assert isinstance(img_tensor, np.ndarray)
+        img_tensor = doc.img_url.load(height=224, width=224)
+        assert img_tensor.shape == (224, 224, 3)
 
-            img_tensor = doc.img_url.load(height=224, width=224)
-            assert img_tensor.shape == (224, 224, 3)
-
-            layout = ('C', 'W', 'H')
-            img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout)
-            assert img_tensor.shape == (3, 200, 100)
+        layout = ('C', 'W', 'H')
+        img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout)
+        assert img_tensor.shape == (3, 200, 100)
+        ```
 
+        ---
 
         :param width: width of the image tensor.
         :param height: height of the image tensor.