docarray · JohannesMessner · Apr 17, 2023 · Apr 7, 2023 · Apr 7, 2023 · Apr 7, 2023
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -101,6 +101,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install poetry
           poetry install --all-extras
+          poetry run pip install elasticsearch==8.6.2
           sudo apt-get update
           sudo apt-get install --no-install-recommends ffmpeg
 
@@ -147,6 +148,7 @@ jobs:
           python -m pip install poetry
           rm poetry.lock
           poetry install --all-extras
+          poetry run pip install elasticsearch==8.6.2
           sudo apt-get update
           sudo apt-get install --no-install-recommends ffmpeg
 

diff --git a/docarray/index/backends/elastic.py b/docarray/index/backends/elastic.py
@@ -58,7 +58,7 @@ class ElasticDocIndex(BaseDocIndex, Generic[TSchema]):
     def __init__(self, db_config=None, **kwargs):
         """Initialize ElasticDocIndex"""
         super().__init__(db_config=db_config, **kwargs)
-        self._db_config = cast(self.DBConfig, self._db_config)
+        self._db_config = cast(ElasticDocIndex.DBConfig, self._db_config)
 
         # ElasticSearch client creation
         if self._db_config.index_name is None:
@@ -406,7 +406,7 @@ def execute_query(self, query: Dict[str, Any], *args, **kwargs) -> Any:
         resp = self._client.search(index=self._index_name, **query)
         docs, scores = self._format_response(resp)
 
-        return _FindResult(documents=docs, scores=scores)
+        return _FindResult(documents=docs, scores=parse_obj_as(NdArray, scores))
 
     def _find(
         self, query: np.ndarray, limit: int, search_field: str = ''
@@ -417,7 +417,7 @@ def _find(
 
         docs, scores = self._format_response(resp)
 
-        return _FindResult(documents=docs, scores=scores)
+        return _FindResult(documents=docs, scores=parse_obj_as(NdArray, scores))
 
     def _find_batched(
         self,
@@ -576,7 +576,7 @@ def _form_text_search_body(
         }
         return body
 
-    def _format_response(self, response: Any) -> Tuple[List[Dict], NdArray]:
+    def _format_response(self, response: Any) -> Tuple[List[Dict], List[Any]]:
         docs = []
         scores = []
         for result in response['hits']['hits']:

diff --git a/docarray/index/backends/elasticv7.py b/docarray/index/backends/elasticv7.py
@@ -3,11 +3,13 @@
 from typing import Any, Dict, List, Optional, Sequence, TypeVar, Union
 
 import numpy as np
+from pydantic import parse_obj_as
 
 from docarray import BaseDoc
 from docarray.index import ElasticDocIndex
 from docarray.index.abstract import BaseDocIndex, _ColumnInfo
 from docarray.typing import AnyTensor
+from docarray.typing.tensor.ndarray import NdArray
 from docarray.utils.find import _FindResult
 
 TSchema = TypeVar('TSchema', bound=BaseDoc)
@@ -120,7 +122,7 @@ def execute_query(self, query: Dict[str, Any], *args, **kwargs) -> Any:
         resp = self._client.search(index=self._index_name, body=query)
         docs, scores = self._format_response(resp)
 
-        return _FindResult(documents=docs, scores=scores)
+        return _FindResult(documents=docs, scores=parse_obj_as(NdArray, scores))
 
     ###############################################
     # Helpers                                     #

diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py
@@ -28,8 +28,8 @@
     _raise_not_supported,
 )
 from docarray.proto import DocProto
-from docarray.typing import NdArray
 from docarray.typing.tensor.abstract_tensor import AbstractTensor
+from docarray.typing.tensor.ndarray import NdArray
 from docarray.utils._internal.misc import import_library, is_np_int
 from docarray.utils.filter import filter_docs
 from docarray.utils.find import _FindResult, _FindResultBatched

diff --git a/docs/.gitignore b/docs/.gitignore
@@ -1,6 +1,7 @@
 api/*
 proto/*
 
-../README.md
-index.md
+<<<<<<< HEAD
+README.md
+#index.md
 CONTRIBUTING.md
diff --git a/docs/api_references/index/backends.md b/docs/api_references/index/backends.md
@@ -0,0 +1,3 @@
+# Backends
+
+::: docarray.index.backends
diff --git a/docs/user_guide/storing/first_step.md b/docs/user_guide/storing/first_step.md
@@ -1,9 +1,9 @@
-# Intro
+# Overview
 
 In the previous sections we saw how to use [`BaseDoc`][docarray.base_doc.doc.BaseDoc], [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] to represent multi-modal data and send it over the wire.
 In this section we will see how to store and persist this data.
 
-DocArray offers to ways of storing your data:
+DocArray offers to ways of storing your data, each of which have their own documentation sections:
 
 1. In a **[Document Store](#document-store)** for simple long-term storage
 2. In a **[Document Index](#document-index)** for fast retrieval using vector similarity
@@ -24,3 +24,20 @@ This section covers the following three topics:
   - [Store on S3](doc_store/store_s3.md)
 
 ## Document Index
+
+A Document Index lets you store your Documents and search through them using vector similarity.
+
+This is useful if you want to store a bunch of data, and at a later point retrieve Documents that are similar to
+some query that you provide.
+Concrete examples where this is relevant are neural search application, Augmenting LLMs and Chatbots with domain knowledge ([Retrieval-Augmented Generation](https://arxiv.org/abs/2005.11401))]),
+or recommender systems.
+
+DocArray's Document Index concept achieves this by providing a unified interface to a number of [vector databases](https://learn.microsoft.com/en-us/semantic-kernel/concepts-ai/vectordb).
+In fact, you can think of Document Index as an **[ORM](https://sqlmodel.tiangolo.com/db-to-code/) for vector databases**.
+
+Currently, DocArray supports the following vector databases:
+
+- [Weaviate](https://weaviate.io/)  |  [Docs](index_weaviate.md)
+- [Qdrant](https://qdrant.tech/)  |  [Docs](index_qdrant.md)
+- [Elasticsearch](https://www.elastic.co/elasticsearch/) v7 and v8  |  [Docs](index_elastic.md)
+- [HNSWlib](https://github.com/nmslib/hnswlib)  |  [Docs](index_hnswlib.md)