Skip to content

feat: redis supports multiple DocumentArrays #540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JoanFM merged 13 commits into from
Sep 22, 2022
Merged

feat: redis supports multiple DocumentArrays #540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
b2ccb52
feat: redis supports storing multiple DAs
AnneYang720 Sep 15, 2022
f40ef7c
docs: modify redis doc for mutiple DAs
AnneYang720 Sep 15, 2022
344c273
Merge branch 'main' into feat-redis-multidas
Sep 16, 2022
45c95d5
refactor: use packaged function
AnneYang720 Sep 16, 2022
3d04d50
docs: clarify expression
AnneYang720 Sep 16, 2022
606297e
perf: batch redis _clear_storage
AnneYang720 Sep 19, 2022
17d0f43
Merge branch 'main' into feat-redis-multidas
Sep 19, 2022
0cc4902
fix: minor error fix
AnneYang720 Sep 20, 2022
173b00e
Merge branch 'feat-redis-multidas' of github.com:AnneYang720/docarray…
AnneYang720 Sep 20, 2022
88301c5
refactor: use scan_iter in redis clear
AnneYang720 Sep 20, 2022
695795e
fix: remove useless imports
AnneYang720 Sep 20, 2022
be798d3
fix: redis delete empty batch
AnneYang720 Sep 20, 2022
37198cb
refactor: use drop_index in redis clear
AnneYang720 Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions docarray/array/storage/redis/backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import copy
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union

import numpy as np
from docarray import Document
from docarray.array.storage.base.backend import BaseBackendMixin, TypeMap
from docarray.helper import dataclass_from_dict
from docarray.helper import dataclass_from_dict, random_identity

from redis import Redis
from redis.commands.search.field import NumericField, TextField, VectorField
Expand All @@ -19,8 +20,7 @@ class RedisConfig:
n_dim: int
host: str = field(default='localhost')
port: int = field(default=6379)
index_name: str = field(default='idx')
flush: bool = field(default=False)
Copy link
Copy Markdown
Member

@JoanFM JoanFM Sep 15, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make sure this is not a breaking change

index_name: Optional[str] = None
update_schema: bool = field(default=True)
distance: str = field(default='COSINE')
redis_config: Dict[str, Any] = field(default_factory=dict)
Expand Down Expand Up @@ -55,6 +55,7 @@ def _init_storage(
config: Optional[Union[RedisConfig, Dict]] = None,
**kwargs,
):
config = copy.deepcopy(config)
if not config:
raise ValueError('Empty config is not allowed for Redis storage')
elif isinstance(config, dict):
Expand All @@ -72,6 +73,9 @@ def _init_storage(
if config.redis_config.get('decode_responses'):
config.redis_config['decode_responses'] = False

if config.index_name is None:
config.index_name = 'index_name__' + random_identity()

self._offset2id_key = config.index_name + '__offset2id'
Copy link
Copy Markdown

@bwanglzu bwanglzu Sep 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we create some constant values?

self._config = config
self.n_dim = self._config.n_dim
Expand All @@ -95,14 +99,10 @@ def _build_client(self):
**self._config.redis_config,
)

if self._config.flush:
client.flushdb()

if self._config.update_schema:
if self._config.index_name.encode() in client.execute_command('FT._LIST'):
client.ft(index_name=self._config.index_name).dropindex()

if self._config.flush or self._config.update_schema:
schema = self._build_schema_from_redis_config()
idef = IndexDefinition(prefix=[self._doc_prefix])
client.ft(index_name=self._config.index_name).create_index(
Expand All @@ -122,7 +122,6 @@ def _ensure_unique_config(
config_joined['index_name'] = (
config_joined['index_name'] + '_subindex_' + subindex_name
)
config_joined['flush'] = False
return config_joined

def _build_schema_from_redis_config(self):
Expand Down
9 changes: 5 additions & 4 deletions docarray/array/storage/redis/getsetdel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
from codecs import unicode_escape_decode
from typing import Dict
from typing import Dict, Iterable, Sequence

from docarray import Document
from docarray.array.storage.base.getsetdel import BaseGetSetDelMixin
from docarray.array.storage.base.helper import Offset2ID
from typing import Sequence, Iterable


class GetSetDelMixin(BaseGetSetDelMixin):
Expand Down Expand Up @@ -120,4 +118,7 @@ def _save_offset2ids(self):
self._update_offset2ids_meta()

def _clear_storage(self):
self._client.flushdb()
self._client.ft(index_name=self._config.index_name).dropindex(
delete_documents=True
)
self._client.delete(self._offset2id_key)
Loading