Skip to content

fix(annlite): bump annlite to v0.4.0 #622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jemmyshin wants to merge 28 commits into from
Closed

fix(annlite): bump annlite to v0.4.0 #622

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
b949459
fix: fix annlite unit test
jemmyshin Oct 14, 2022
0e4b335
fix: fix annlite unit test
jemmyshin Oct 14, 2022
1e0e3cb
fix: fix annlite unit test
jemmyshin Oct 14, 2022
9616ce7
fix: resolve file lock in rocksdb
jemmyshin Oct 17, 2022
370df2d
fix: close da in plot test
jemmyshin Oct 17, 2022
468888c
fix: close da in plot test
jemmyshin Oct 17, 2022
71fdcdc
fix: remove close in backend
jemmyshin Oct 19, 2022
7a02127
feat: add annlite ctx mngr
jemmyshin Oct 20, 2022
8ca4027
feat: add annlite ctx mngr
jemmyshin Oct 20, 2022
ab3846f
fix: fix plot test
jemmyshin Oct 20, 2022
8026e87
fix: fix plot test
jemmyshin Oct 20, 2022
ad5c2f1
fix: fix annlite based unit test
jemmyshin Oct 20, 2022
1abdaa2
fix: ensure ci use main branch of annlite
jemmyshin Oct 20, 2022
7c7c648
fix: ensure ci use main branch of annlite
jemmyshin Oct 20, 2022
b99d438
fix: ensure ci use main branch of annlite
jemmyshin Oct 20, 2022
0665f4b
fix: remove annlite cntx mngr
jemmyshin Oct 21, 2022
21e6d1d
fix: recover annlite ci
jemmyshin Oct 21, 2022
d8bc517
fix: revert changes in tests
jemmyshin Oct 21, 2022
99cfb70
fix: revert changes in tests
jemmyshin Oct 21, 2022
669aea0
Merge branch 'main' into fix-annlite
jemmyshin Oct 21, 2022
cde4a16
fix: close annlite after iteration
jemmyshin Oct 21, 2022
e1274aa
fix: close annlite after iteration
jemmyshin Oct 21, 2022
089e316
fix: remove redundant cntx mngr
jemmyshin Oct 21, 2022
95d054b
fix: fix and enable the annlite test
jemmyshin Oct 21, 2022
d622fdb
ci: remove pip install --pre annlite
jemmyshin Oct 21, 2022
8176720
Merge branch 'main' into fix-annlite
jemmyshin Oct 24, 2022
d15dce0
Merge branch 'main' into fix-annlite
jemmyshin Oct 24, 2022
49c7ea6
Merge branch 'main' into fix-annlite
jemmyshin Oct 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions tests/unit/array/mixins/test_content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ def test_content_empty_getter_return_none(cls, content_attr, start_storage):
da = cls()
assert getattr(da, content_attr) is None

if cls == DocumentArrayAnnlite:
Copy link
Copy Markdown
Member

@alaeddine-13 alaeddine-13 Oct 21, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for example here, DA will create an annlite instance with temp file name. I don't think anything would access the same temp file concurrently, why do we need to release the lock ?

Copy link
Copy Markdown
Contributor Author

@jemmyshin jemmyshin Oct 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is because we need to test with different parameters, it's like a for loop, need to close it before another iteration.

da._annlite.close()


@pytest.mark.parametrize(
'cls',
Expand Down Expand Up @@ -77,6 +80,9 @@ def test_content_empty_setter(cls, content_attr, start_storage):
setattr(da, content_attr[0], content_attr[1])
assert getattr(da, content_attr[0]) is None

if cls == DocumentArrayAnnlite:
da._annlite.close()


@pytest.mark.parametrize(
'cls,config',
Expand Down Expand Up @@ -111,6 +117,9 @@ def test_content_getter_setter(cls, content_attr, config, start_storage):
da.contents = None
assert da.contents is None

if cls == DocumentArrayAnnlite:
da._annlite.close()


@pytest.mark.parametrize('da_len', [0, 1, 2])
@pytest.mark.parametrize(
Expand Down Expand Up @@ -149,6 +158,9 @@ def test_content_empty(da_len, da_cls, config, start_storage):
assert not da.tensors
assert da.blobs == [b''] * da_len

if da_cls == DocumentArrayAnnlite:
da._annlite.close()


@pytest.mark.parametrize('da_len', [0, 1, 2])
@pytest.mark.parametrize(
Expand All @@ -171,3 +183,6 @@ def test_embeddings_setter(da_len, da_cls, config, start_storage):
da.embeddings = np.random.rand(da_len, 5)
for doc in da:
assert doc.embedding.shape == (5,)

if da_cls == DocumentArrayAnnlite:
da._annlite.close()
3 changes: 3 additions & 0 deletions tests/unit/array/mixins/test_empty.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ def test_empty_non_zero(da_cls, config, start_storage):

assert len(da) == 0

if da_cls == DocumentArrayAnnlite:
da._annlite.close()

# Assert .empty provides a da of the correct length
if config:
da = da_cls.empty(10, config=config)
Expand Down
34 changes: 28 additions & 6 deletions tests/unit/array/mixins/test_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,20 +43,31 @@ def test_document_save_load(
):
tmp_file = os.path.join(tmp_path, 'test')
da = da_cls(docs, config=config())

da.insert(2, Document(id='new'))
da.save(tmp_file, file_format=method, encoding=encoding)

da_info = {
'id': [d.id for d in da],
'embedding': [d.embedding for d in da],
'content': [d.content for d in da],
}

if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da_r = type(da).load(
tmp_file, file_format=method, encoding=encoding, config=config()
)

assert type(da) is type(da_r)
assert len(da) == len(da_r)
assert len(da) == len(da_info['id'])
assert da_r[2].id == 'new'
for d, d_r in zip(da, da_r):
assert d.id == d_r.id
np.testing.assert_equal(d.embedding, d_r.embedding)
assert d.content == d_r.content

for idx, d_r in enumerate(da_r):
assert da_info['id'][idx] == d_r.id
np.testing.assert_equal(da_info['embedding'][idx], d_r.embedding)
assert da_info['content'][idx] == d_r.content


@pytest.mark.parametrize('flatten_tags', [True, False])
Expand Down Expand Up @@ -200,13 +211,24 @@ def test_from_to_pd_dataframe(da_cls, config, start_storage):
)
def test_from_to_bytes(da_cls, config, start_storage):
# simple
assert len(da_cls.load_binary(bytes(da_cls.empty(2, config=config)))) == 2
if da_cls == DocumentArrayAnnlite:
da = da_cls.empty(2, config=config)
da_bytes = da.to_bytes()
da._annlite.close()

db = da_cls.from_bytes(da_bytes, config=config)
assert len(db) == 2
db._annlite.close()
else:
assert len(da_cls.load_binary(bytes(da_cls.empty(2, config=config)))) == 2

da = da_cls.empty(2, config=config)

da[:, 'embedding'] = [[1, 2, 3], [4, 5, 6]]
da[:, 'tensor'] = [[1, 2], [2, 1]]
da[0, 'tags'] = {'hello': 'world'}
if da_cls == DocumentArrayAnnlite:
Copy link
Copy Markdown
Member

@JoanFM JoanFM Oct 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove this

da._annlite.close()
da2 = da_cls.load_binary(bytes(da))
assert da2.tensors == [[1, 2], [2, 1]]
import numpy as np
Expand Down
6 changes: 4 additions & 2 deletions tests/unit/array/mixins/test_plot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@
[
(DocumentArray, None),
(DocumentArraySqlite, None),
# TODO: restore this after annlite issue is fixed in #622
# (DocumentArrayAnnlite, AnnliteConfig(n_dim=128)),
(DocumentArrayAnnlite, AnnliteConfig(n_dim=128)),
# (DocumentArrayWeaviate, WeaviateConfig(n_dim=128)),
(DocumentArrayQdrant, QdrantConfig(n_dim=128, scroll_batch_size=8)),
(DocumentArrayElastic, ElasticConfig(n_dim=128)),
Expand Down Expand Up @@ -57,6 +56,9 @@ def test_sprite_fail_tensor_success_uri(
da.save_gif(tmpdir / 'sprint_da.gif', show_index=show_index, channel_axis=0)
assert os.path.exists(tmpdir / 'sprint_da.png')

if da_cls == DocumentArrayAnnlite:
da._annlite.close()


@pytest.mark.parametrize('image_source', ['tensor', 'uri'])
@pytest.mark.parametrize(
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/array/mixins/test_text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ def test_da_vocabulary(da_cls, config, docs, min_freq, start_storage):
assert not vocab.values()
assert not vocab.keys()

if da_cls == DocumentArrayAnnlite:
da._annlite.close()


@pytest.mark.parametrize(
'da_cls,config',
Expand Down
12 changes: 11 additions & 1 deletion tests/unit/array/test_advance_indexing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,9 @@ def test_sequence_int(docs, nparray, storage, config, start_storage):
assert docs[5].text == 'new'
assert docs[9].text == 'new'

if storage == 'annlite':
docs._annlite.close()


@pytest.mark.parametrize(
'storage,config',
Expand Down Expand Up @@ -705,16 +708,23 @@ def test_offset2ids_persistence(storage, config, start_storage):
assert da_ids == [str(i) for i in range(5)]
da.sync()

if storage == 'annlite':
da._annlite.close()

da1 = DocumentArray(storage=storage, config=config)

assert da1[:, 'id'] == da_ids

with da1:
da1.extend([Document(id=i) for i in 'abc'])
da1_ids = da1[:, 'id']
assert len(da1) == 8

if storage == 'annlite':
da1._annlite.close()

da2 = DocumentArray(storage=storage, config=config)
assert da2[:, 'id'] == da1[:, 'id']
assert da2[:, 'id'] == da1_ids


def test_dam_conflicting_ids():
Expand Down
20 changes: 20 additions & 0 deletions tests/unit/array/test_construct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,33 +28,53 @@ def test_construct_docarray(da_cls, config, start_storage):
if config:
da = da_cls(config=config)
assert len(da) == 0
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls(Document(), config=config)
assert len(da) == 1
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls([Document(), Document()], config=config)
assert len(da) == 2
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls((Document(), Document()), config=config)
assert len(da) == 2
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls((Document() for _ in range(10)), config=config)
assert len(da) == 10
if da_cls == DocumentArrayAnnlite:
da._annlite.close()
else:
da = da_cls()
assert len(da) == 0
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls(Document())
assert len(da) == 1
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls([Document(), Document()])
assert len(da) == 2
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls((Document(), Document()))
assert len(da) == 2
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

da = da_cls((Document() for _ in range(10)))
assert len(da) == 10
if da_cls == DocumentArrayAnnlite:
da._annlite.close()

if da_cls is DocumentArrayInMemory:
da1 = da_cls(da)
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/document/test_plot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ def test_matches_sprites(
da[0].plot_matches_sprites(top_k, output=tmpdir / 'sprint_da.png')
assert os.path.exists(tmpdir / 'sprint_da.png')

if da_cls == DocumentArrayAnnlite:
das._annlite.close()


@pytest.mark.parametrize('image_source', ['tensor', 'uri'])
@pytest.mark.parametrize(
Expand Down