diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9715db93d72..d289509f5d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: matrix: python-version: [3.7] # test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} - test-path: [tests/integrations, tests/units] + test-path: [tests/integrations, tests/units, tests/documentation] steps: - uses: actions/checkout@v2.5.0 - name: Set up Python ${{ matrix.python-version }} @@ -144,7 +144,7 @@ jobs: fail-fast: false matrix: python-version: [3.7] - test-path: [tests/integrations, tests/units] + test-path: [tests/integrations, tests/units, tests/documentation] steps: - uses: actions/checkout@v2.5.0 - name: Set up Python ${{ matrix.python-version }} @@ -275,7 +275,6 @@ jobs: poetry run pytest -m 'benchmark' tests timeout-minutes: 30 - # just for blocking the merge until all parallel core-test are successful success-all-test: needs: [docarray-test, docarray-test-proto3, docarray-doc-index, docarray-test-tensorflow, docarray-test-benchmarks, import-test, check-black, check-mypy, lint-ruff] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a7c5542f3e..bccbe2f206d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,4 +23,13 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit rev: v0.0.243 hooks: - - id: ruff \ No newline at end of file + - id: ruff + +- repo: https://github.com/asottile/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + args: + - -S + additional_dependencies: + - black==22.3.0 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e06a3b9df21..332daa90ca4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -276,6 +276,7 @@ This allows: If you need to monitor and debug your code, you can enable docarray logging: ```python import logging + logging.getLogger('docarray').setLevel(logging.DEBUG) ``` diff --git a/docs/api_references/.pages b/docs/api_references/.pages new file mode 100644 index 00000000000..2ac040fecf7 --- /dev/null +++ b/docs/api_references/.pages @@ -0,0 +1,4 @@ +nav: + - ... + +sort_type: natural \ No newline at end of file diff --git a/docs/assets/logo-dark.svg b/docs/assets/logo-dark.svg index 5c3d8d504a8..db2420c6673 100644 --- a/docs/assets/logo-dark.svg +++ b/docs/assets/logo-dark.svg @@ -1,12 +1,20 @@ - docarray-logo_text-dark - - - + docarray-logo_text-light + + + + - + + + + + + + + \ No newline at end of file diff --git a/docs/assets/logo-light.svg b/docs/assets/logo-light.svg index db2420c6673..5c3d8d504a8 100644 --- a/docs/assets/logo-light.svg +++ b/docs/assets/logo-light.svg @@ -1,20 +1,12 @@ - docarray-logo_text-light - - - - + docarray-logo_text-dark + + + - - - - - - - - + \ No newline at end of file diff --git a/docs/tutorials/add_doc_index.md b/docs/how_to/add_doc_index.md similarity index 100% rename from docs/tutorials/add_doc_index.md rename to docs/how_to/add_doc_index.md diff --git a/docs/tutorials/multimodal_training_and_serving.md b/docs/how_to/multimodal_training_and_serving.md similarity index 100% rename from docs/tutorials/multimodal_training_and_serving.md rename to docs/how_to/multimodal_training_and_serving.md diff --git a/docs/tutorials/optimize_performance_with_id_generation.md b/docs/how_to/optimize_performance_with_id_generation.md similarity index 100% rename from docs/tutorials/optimize_performance_with_id_generation.md rename to docs/how_to/optimize_performance_with_id_generation.md diff --git a/docs/user_guide/first_step.md b/docs/user_guide/first_step.md new file mode 100644 index 00000000000..0671e3a096a --- /dev/null +++ b/docs/user_guide/first_step.md @@ -0,0 +1 @@ +# First Step : BaseDoc diff --git a/docs/user_guide/intro.md b/docs/user_guide/intro.md new file mode 100644 index 00000000000..c500c92629f --- /dev/null +++ b/docs/user_guide/intro.md @@ -0,0 +1 @@ +# User Guide - Intro diff --git a/mkdocs.yml b/mkdocs.yml index 7ab61cfbe22..e7749bc2874 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,13 @@ -site_name: "DocArray" +site_name: DocArray Documentation +site_description: DocArray, DocArray is a library for representing, sending and storing multi-modal data, with a focus on applications in ML and Neural Search. +site_url: https://docs.docarray.org/ +repo_name: docarray/docarray +repo_url: https://github.com/docarray/docarray +edit_uri: '' theme: -# logo: assets/favicon.png + logo: assets/logo-light.svg + + favicon: assets/favicon.png name: material features: # - navigation.sections @@ -11,16 +18,17 @@ theme: primary: teal toggle: icon: material/brightness-7 - - name: Switch to dark mode + # Palette toggle for dark mode - scheme: slate primary: teal toggle: icon: material/brightness-4 name: Switch to light mode +# logo: assets/docarray-light.svg + #primary: teal markdown_extensions: - pymdownx.highlight: @@ -40,12 +48,37 @@ markdown_extensions: - attr_list - md_in_html +extra: +# analytics: +# provider: google + social: + - icon: fontawesome/brands/github-alt + link: https://github.com/docarray/docarray + - icon: fontawesome/brands/discord + link: https://discord.com/invite/WaMp6PVPgR + - icon: fontawesome/brands/twitter + link: https://twitter.com/docarray + plugins: - search + - awesome-pages - mkdocstrings: handlers: python: options: docstring_style: sphinx - inherited_members: true \ No newline at end of file + inherited_members: true + +nav: + - Home: README.md + - Tutorial - User Guide: + - user_guide/intro.md + - user_guide/first_step.md + + - How-to: + - how_to/add_doc_index.md + - how_to/multimodal_training_and_serving.md + - how_to/optimize_performance_with_id_generation.md + - ... + - Contributing: CONTRIBUTING.md diff --git a/poetry.lock b/poetry.lock index 6b31ed3d618..2f89d1231dc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "aiohttp" @@ -410,6 +410,21 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "blacken-docs" +version = "1.13.0" +description = "Run Black on Python code blocks in documentation files." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "blacken_docs-1.13.0-py3-none-any.whl", hash = "sha256:455388df506fca04742f36a4a3475630eb7f141cb98acc6070d3c24bcf69cdda"}, + {file = "blacken_docs-1.13.0.tar.gz", hash = "sha256:2babba84a42fb31a1d393dcf5a9a66d9b0657bdc320aec69d9f96301501dba35"}, +] + +[package.dependencies] +black = ">=22.1.0" + [[package]] name = "bleach" version = "5.0.1" @@ -470,6 +485,18 @@ urllib3 = ">=1.25.4,<1.27" [package.extras] crt = ["awscrt (==0.16.9)"] +[[package]] +name = "bracex" +version = "2.3.post1" +description = "Bash style brace expander." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "bracex-2.3.post1-py3-none-any.whl", hash = "sha256:351b7f20d56fb9ea91f9b9e9e7664db466eb234188c175fd943f8f755c807e73"}, + {file = "bracex-2.3.post1.tar.gz", hash = "sha256:e7b23fc8b2cd06d3dec0692baabecb249dda94e06a617901ff03a6c56fd71693"}, +] + [[package]] name = "cached-property" version = "1.5.2" @@ -1658,6 +1685,23 @@ files = [ Markdown = ">=3.3" mkdocs = ">=1.1" +[[package]] +name = "mkdocs-awesome-pages-plugin" +version = "2.8.0" +description = "An MkDocs plugin that simplifies configuring page titles and their order" +category = "dev" +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "mkdocs-awesome-pages-plugin-2.8.0.tar.gz", hash = "sha256:af7e327e14b2eea3b2735c37428e33a528ecd2d9ae2296dc0f1632f0f3bc28f7"}, + {file = "mkdocs_awesome_pages_plugin-2.8.0-py3-none-any.whl", hash = "sha256:6b21ad4f41aecbe89e3a9a51f8837892cc7ce8ca0f9f4e0a355d56159ace3d68"}, +] + +[package.dependencies] +mkdocs = ">=1" +natsort = ">=8.1.0" +wcmatch = ">=7" + [[package]] name = "mkdocs-material" version = "9.1.3" @@ -1735,6 +1779,21 @@ files = [ griffe = ">=0.24" mkdocstrings = ">=0.19" +[[package]] +name = "mktestdocs" +version = "0.2.0" +description = "" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "mktestdocs-0.2.0-py2.py3-none-any.whl", hash = "sha256:0ce2ba702dfe7f2a516878fd1787d4c1f95e4a088248893bb5788ac037010559"}, + {file = "mktestdocs-0.2.0.tar.gz", hash = "sha256:a6b401c63ac02ab683443e0fcb27c58fc8c2264cf4e9e93835741d234f917267"}, +] + +[package.extras] +test = ["pytest (>=4.0.2)"] + [[package]] name = "multidict" version = "6.0.4" @@ -1879,6 +1938,22 @@ files = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] +[[package]] +name = "natsort" +version = "8.3.1" +description = "Simple yet flexible natural sorting in Python." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "natsort-8.3.1-py3-none-any.whl", hash = "sha256:d583bc9050dd10538de36297c960b93f873f0cd01671a3c50df5bd86dd391dcb"}, + {file = "natsort-8.3.1.tar.gz", hash = "sha256:517595492dde570a4fd6b6a76f644440c1ba51e2338c8a671d7f0475fda8f9fd"}, +] + +[package.extras] +fast = ["fastnumbers (>=2.0.0)"] +icu = ["PyICU (>=1.0.0)"] + [[package]] name = "nbclassic" version = "0.4.8" @@ -3762,6 +3837,21 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "wcmatch" +version = "8.4.1" +description = "Wildcard/glob file name matcher." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4.1-py3-none-any.whl", hash = "sha256:3476cd107aba7b25ba1d59406938a47dc7eec6cfd0ad09ff77193f21a964dee7"}, + {file = "wcmatch-8.4.1.tar.gz", hash = "sha256:b1f042a899ea4c458b7321da1b5e3331e3e0ec781583434de1301946ceadb943"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + [[package]] name = "wcwidth" version = "0.2.5" @@ -3926,7 +4016,8 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [extras] audio = ["pydub"] aws = ["smart-open"] -common = ["lz4", "protobuf"] +common = ["protobuf", "lz4"] +full = ["protobuf", "lz4", "pillow", "types-pillow", "av", "pydub", "trimesh"] hnswlib = ["hnswlib"] image = ["pillow", "types-pillow"] jac = ["jina-hubble-sdk"] @@ -3939,4 +4030,4 @@ web = ["fastapi"] [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "0872bd8654de67d349699a227cd2dc1708c6fa5066c84e1f295d630184ee5ac4" +content-hash = "c5b13c9b48aa9edf9d494ce8ba91cfdd9f78d4220ae758ac8a74b69963fc7253" diff --git a/pyproject.toml b/pyproject.toml index 61b10f7f298..9e30c3a804e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,9 @@ pandas = ["pandas"] jac = ["jina-hubble-sdk"] aws = ["smart-open"] +# all +full = ["protobuf", "lz4", "pillow", "types-pillow", "av", "pydub", "trimesh"] + [tool.poetry.dev-dependencies] pytest = ">=7.0" pre-commit = ">=2.20.0" @@ -49,6 +52,7 @@ types-protobuf = ">=3.20.4" black = ">=22.10.0" isort = ">=5.10.1" ruff = ">=0.0.243" +blacken-docs = ">=1.13.0" [tool.poetry.group.dev.dependencies] uvicorn = ">=0.19.0" @@ -59,7 +63,8 @@ pytest-asyncio = ">=0.20.2" [tool.poetry.group.docs.dependencies] mkdocstrings = {extras = ["python"], version = ">=0.20.0"} mkdocs-material= ">=9.1.2" - +mkdocs-awesome-pages-plugin = ">=2.8.0" +mktestdocs= ">=0.2.0" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/tests/documentation/test_docs.py b/tests/documentation/test_docs.py new file mode 100644 index 00000000000..4eceb252f89 --- /dev/null +++ b/tests/documentation/test_docs.py @@ -0,0 +1,13 @@ +import pathlib + +import pytest +from mktestdocs import check_md_file + + +# @pytest.mark.parametrize('fpath', pathlib.Path("docs").glob("**/*.md"), ids=str) +# to use later +@pytest.mark.parametrize( + 'fpath', pathlib.Path('docs/user_guide').glob('**/*.md'), ids=str +) +def test_files_good(fpath): + check_md_file(fpath=fpath, memory=True)