From b5f14bc223f47246f6f62c525f941ddebd59c32d Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:07:47 +0800 Subject: [PATCH 01/11] ci: fix the wrong error msg Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 283 +++++++++++++++++++-------------------- 1 file changed, 141 insertions(+), 142 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1853247e9f9..bceedd80d34 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -7,145 +7,147 @@ on: jobs: - prep-testbed: - if: | - !startsWith(github.event.head_commit.message, 'chore') && - !startsWith(github.event.head_commit.message, 'build: hotfix') && - !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.5.0 - - id: set-matrix - run: | - sudo apt-get install jq - export value=$(bash scripts/get-all-test-paths.sh) - echo "matrix=$value" >> $GITHUB_OUTPUT - export value=$(bash scripts/get-all-oldproto-test-paths.sh) - echo "matrix-oldproto=$value" >> $GITHUB_OUTPUT - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - matrix-oldproto: ${{ steps.set-matrix.outputs.matrix-oldproto }} - - docarray-test: - needs: prep-testbed - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [3.7] - test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} - steps: - - uses: actions/checkout@v2.5.0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Prepare enviroment - run: | - python -m pip install --upgrade pip - python -m pip install wheel - # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" - pip install --no-cache-dir ".[test]" - pip install --no-cache-dir ".[qdrant]" - pip install --no-cache-dir ".[annlite]" - pip install --no-cache-dir ".[weaviate]" - pip install --no-cache-dir ".[elasticsearch]" - pip install --no-cache-dir ".[redis]" - pip install --no-cache-dir ".[full]" - sudo apt-get install libsndfile1 - - name: Test - id: test - run: | - pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ - -v -s -m "not gpu" ${{ matrix.test-path }} - echo "codecov_flag=docarray" >> $GITHUB_OUTPUT - timeout-minutes: 60 - env: - JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" - - name: Check codecov file - id: check_files - uses: andstor/file-existence-action@v1 - with: - files: "coverage.xml" - - name: Upload coverage from test to Codecov - uses: codecov/codecov-action@v3.1.1 - if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' - with: - file: coverage.xml - flags: ${{ steps.test.outputs.codecov_flag }} - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - - docarray-oldproto-test: - needs: prep-testbed - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [3.7] - test-path: ${{fromJson(needs.prep-testbed.outputs.matrix-oldproto)}} - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Prepare enviroment - run: | - python -m pip install --upgrade pip - python -m pip install wheel - # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" - pip install --no-cache-dir ".[test]" - pip install --no-cache-dir ".[qdrant]" - pip install --no-cache-dir ".[annlite]" - pip install --no-cache-dir ".[weaviate]" - pip install --no-cache-dir ".[elasticsearch]" - pip install --no-cache-dir ".[redis]" - pip install --no-cache-dir ".[full]" - pip install --no-cache-dir "protobuf<3.20.0" - sudo apt-get install libsndfile1 - - name: Test - id: test - run: | - pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ - -v -s -m "not gpu" ${{ matrix.test-path }} - echo "::set-output name=codecov_flag::docarray" - timeout-minutes: 60 - env: - JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" - - name: Check codecov file - id: check_files - uses: andstor/file-existence-action@v1 - with: - files: "coverage.xml" - - name: Upload coverage from test to Codecov - uses: codecov/codecov-action@v2 - if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' - with: - file: coverage.xml - flags: ${{ steps.test.outputs.codecov_flag }} - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - - prerelease: - needs: [docarray-test, docarray-oldproto-test] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.5.0 - with: - fetch-depth: 100 - - name: Pre-release (.devN) - run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - pip install twine wheel - ./scripts/release.sh - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} +# prep-testbed: +# if: | +# !startsWith(github.event.head_commit.message, 'chore') && +# !startsWith(github.event.head_commit.message, 'build: hotfix') && +# !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2.5.0 +# - id: set-matrix +# run: | +# sudo apt-get install jq +# export value=$(bash scripts/get-all-test-paths.sh) +# echo "matrix=$value" >> $GITHUB_OUTPUT +# export value=$(bash scripts/get-all-oldproto-test-paths.sh) +# echo "matrix-oldproto=$value" >> $GITHUB_OUTPUT +# outputs: +# matrix: ${{ steps.set-matrix.outputs.matrix }} +# matrix-oldproto: ${{ steps.set-matrix.outputs.matrix-oldproto }} +# +# docarray-test: +# needs: prep-testbed +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: [3.7] +# test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} +# steps: +# - uses: actions/checkout@v2.5.0 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Prepare enviroment +# run: | +# python -m pip install --upgrade pip +# python -m pip install wheel +# # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" +# pip install --no-cache-dir ".[test]" +# pip install --no-cache-dir ".[qdrant]" +# pip install --no-cache-dir ".[annlite]" +# pip install --no-cache-dir ".[weaviate]" +# pip install --no-cache-dir ".[elasticsearch]" +# pip install --no-cache-dir ".[redis]" +# pip install --no-cache-dir ".[full]" +# sudo apt-get install libsndfile1 +# - name: Test +# id: test +# run: | +# pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ +# -v -s -m "not gpu" ${{ matrix.test-path }} +# echo "codecov_flag=docarray" >> $GITHUB_OUTPUT +# timeout-minutes: 60 +# env: +# JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" +# - name: Check codecov file +# id: check_files +# uses: andstor/file-existence-action@v1 +# with: +# files: "coverage.xml" +# - name: Upload coverage from test to Codecov +# uses: codecov/codecov-action@v3.1.1 +# if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' +# with: +# file: coverage.xml +# flags: ${{ steps.test.outputs.codecov_flag }} +# fail_ci_if_error: false +# token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos +# +# docarray-oldproto-test: +# needs: prep-testbed +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: [3.7] +# test-path: ${{fromJson(needs.prep-testbed.outputs.matrix-oldproto)}} +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Prepare enviroment +# run: | +# python -m pip install --upgrade pip +# python -m pip install wheel +# # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" +# pip install --no-cache-dir ".[test]" +# pip install --no-cache-dir ".[qdrant]" +# pip install --no-cache-dir ".[annlite]" +# pip install --no-cache-dir ".[weaviate]" +# pip install --no-cache-dir ".[elasticsearch]" +# pip install --no-cache-dir ".[redis]" +# pip install --no-cache-dir ".[full]" +# pip install --no-cache-dir "protobuf<3.20.0" +# sudo apt-get install libsndfile1 +# - name: Test +# id: test +# run: | +# pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ +# -v -s -m "not gpu" ${{ matrix.test-path }} +# echo "::set-output name=codecov_flag::docarray" +# timeout-minutes: 60 +# env: +# JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" +# - name: Check codecov file +# id: check_files +# uses: andstor/file-existence-action@v1 +# with: +# files: "coverage.xml" +# - name: Upload coverage from test to Codecov +# uses: codecov/codecov-action@v2 +# if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' +# with: +# file: coverage.xml +# flags: ${{ steps.test.outputs.codecov_flag }} +# fail_ci_if_error: false +# token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos +# +# prerelease: +# needs: [docarray-test, docarray-oldproto-test] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2.5.0 +# with: +# fetch-depth: 100 +# - name: Pre-release (.devN) +# run: | +# git fetch --depth=1 origin +refs/tags/*:refs/tags/* +# pip install twine wheel +# ./scripts/release.sh +# env: +# TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} +# TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} +# JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} check-readme-modification: runs-on: ubuntu-latest + outputs: + readme_changed: ${{ steps.step_output.outputs.readme_changed }} steps: - uses: actions/checkout@v3 with: @@ -159,16 +161,13 @@ jobs: README.md - name: Check if README is modified + id: step_output if: steps.changed-files-specific.outputs.any_changed == 'true' run: | - echo "README.md is modified" - touch SUCCESS - - - name: Return outputs - run: | - [[ -f SUCCESS ]] + echo "readme_changed=true" >> $GITHUB_OUTPUT publish-docarray-org: needs: check-readme-modification + if: jobs.check-readme-modification.outputs.readme_changed == "true" uses: ./.github/workflows/publish-docarray-org.yml secrets: inherit \ No newline at end of file From 0dc5387a935bb6f1d870954862a8964996df5f85 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:09:43 +0800 Subject: [PATCH 02/11] ci: add debugging Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index bceedd80d34..8b6f92d4dbb 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -2,8 +2,8 @@ name: CD on: push: - branches: - - main +# branches: +# - main jobs: From 37bf8965f3f79f3eb1f69d4b7bd41619e68073e9 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:16:52 +0800 Subject: [PATCH 03/11] ci: fix the gramma Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8b6f92d4dbb..9a52b061af4 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -146,8 +146,6 @@ jobs: check-readme-modification: runs-on: ubuntu-latest - outputs: - readme_changed: ${{ steps.step_output.outputs.readme_changed }} steps: - uses: actions/checkout@v3 with: @@ -161,13 +159,6 @@ jobs: README.md - name: Check if README is modified - id: step_output if: steps.changed-files-specific.outputs.any_changed == 'true' - run: | - echo "readme_changed=true" >> $GITHUB_OUTPUT - - publish-docarray-org: - needs: check-readme-modification - if: jobs.check-readme-modification.outputs.readme_changed == "true" - uses: ./.github/workflows/publish-docarray-org.yml - secrets: inherit \ No newline at end of file + uses: ./.github/workflows/publish-docarray-org.yml + secrets: inherit \ No newline at end of file From 793ecac8e212f06aaa131da4cc2d686b2377c4c3 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:19:25 +0800 Subject: [PATCH 04/11] ci: fix the gramma Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9a52b061af4..f50aeadf7fa 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -146,6 +146,7 @@ jobs: check-readme-modification: runs-on: ubuntu-latest + secrets: inherit steps: - uses: actions/checkout@v3 with: @@ -160,5 +161,4 @@ jobs: - name: Check if README is modified if: steps.changed-files-specific.outputs.any_changed == 'true' - uses: ./.github/workflows/publish-docarray-org.yml - secrets: inherit \ No newline at end of file + uses: ./.github/workflows/publish-docarray-org.yml \ No newline at end of file From cf3f11361557678718e95549c7a3520dd66d7445 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:24:48 +0800 Subject: [PATCH 05/11] ci: fix the gramma Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f50aeadf7fa..dbe5a205f90 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -146,7 +146,6 @@ jobs: check-readme-modification: runs-on: ubuntu-latest - secrets: inherit steps: - uses: actions/checkout@v3 with: From bbd741fe9452a8b545f3a92722859c91b5b58207 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:27:02 +0800 Subject: [PATCH 06/11] chore: clean up readme Signed-off-by: Nan Wang --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b9c7aaa1f3a..2595f2fcff6 100644 --- a/README.md +++ b/README.md @@ -167,10 +167,10 @@ Let's use DocArray and the [Totally Looks Like](https://sites.google.com/view/to - Visualizing top-9 matches using DocArray API - Visualizing top-9 matches using DocArray API - Visualizing top-9 matches using DocArray API - Visualizing top-9 matches using DocArray API + Visualizing top-9 matches using DocArray API + Visualizing top-9 matches using DocArray API + Visualizing top-9 matches using DocArray API + Visualizing top-9 matches using DocArray API From bd2efcd4817ae34a582eade394c064a0a34494c5 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:37:32 +0800 Subject: [PATCH 07/11] ci: fix the gramma Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index dbe5a205f90..de0da37e66d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -146,6 +146,8 @@ jobs: check-readme-modification: runs-on: ubuntu-latest + outputs: + readme_changed: ${{ steps.step_output.outputs.readme_changed }} steps: - uses: actions/checkout@v3 with: @@ -159,5 +161,12 @@ jobs: README.md - name: Check if README is modified + id: step_output if: steps.changed-files-specific.outputs.any_changed == 'true' - uses: ./.github/workflows/publish-docarray-org.yml \ No newline at end of file + run: | + echo "readme_changed=true" >> $GITHUB_OUTPUT + + publish-docarray-org: + needs: check-readme-modification.outputs.readme_changed + uses: ./.github/workflows/publish-docarray-org.yml + secrets: inherit \ No newline at end of file From 3f2a548087f19ea3526a1fc1d8d83aaf88d70657 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:42:09 +0800 Subject: [PATCH 08/11] ci: fix the gramma Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index de0da37e66d..fb76a12a01e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -167,6 +167,7 @@ jobs: echo "readme_changed=true" >> $GITHUB_OUTPUT publish-docarray-org: - needs: check-readme-modification.outputs.readme_changed + needs: check-readme-modification + if: needs.check-readme-modification.outputs.readme_changed == 'true' uses: ./.github/workflows/publish-docarray-org.yml secrets: inherit \ No newline at end of file From 73c8ea76fb9af786cfe30190db3e80fdece2f729 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:43:55 +0800 Subject: [PATCH 09/11] chore: clean up readme Signed-off-by: Nan Wang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2595f2fcff6..973a276e235 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,7 @@ left_da.plot_image_sprites() ```

-Load totally looks like dataset with docarray API +Load totally looks like dataset with docarray API

### Apply preprocessing From c8fa58762252f3ded568822d32aafb29e562755d Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:47:00 +0800 Subject: [PATCH 10/11] chore: clean up readme Signed-off-by: Nan Wang --- .github/workflows/publish-docarray-org.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-docarray-org.yml b/.github/workflows/publish-docarray-org.yml index 8e52c03fa7f..23250e7b188 100644 --- a/.github/workflows/publish-docarray-org.yml +++ b/.github/workflows/publish-docarray-org.yml @@ -31,7 +31,7 @@ jobs: - name: Check trigger event run: | touch SUCCESS - if: github.event_name == 'pull_request' + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} - name: Fail release token run: | [[ -f SUCCESS ]] diff --git a/README.md b/README.md index 973a276e235..cbb79461ed8 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ left_da.plot_embeddings(image_sprites=True) ```

-Visualizing embedding via tSNE and embedding projector +Visualizing embedding via tSNE and embedding projector

Fun is fun, but recall our goal is to match left images against right images and so far we have only handled the left. Let's repeat the same procedure for the right: From fae608544c9e0acedbf84ed3ec13d4e9afc3d641 Mon Sep 17 00:00:00 2001 From: Nan Wang Date: Thu, 1 Dec 2022 23:49:36 +0800 Subject: [PATCH 11/11] ci: remove the debugging lines Signed-off-by: Nan Wang --- .github/workflows/cd.yml | 276 +++++++++++++++++++-------------------- 1 file changed, 138 insertions(+), 138 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index fb76a12a01e..ccd020109e1 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -2,147 +2,147 @@ name: CD on: push: -# branches: -# - main + branches: + - main jobs: -# prep-testbed: -# if: | -# !startsWith(github.event.head_commit.message, 'chore') && -# !startsWith(github.event.head_commit.message, 'build: hotfix') && -# !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2.5.0 -# - id: set-matrix -# run: | -# sudo apt-get install jq -# export value=$(bash scripts/get-all-test-paths.sh) -# echo "matrix=$value" >> $GITHUB_OUTPUT -# export value=$(bash scripts/get-all-oldproto-test-paths.sh) -# echo "matrix-oldproto=$value" >> $GITHUB_OUTPUT -# outputs: -# matrix: ${{ steps.set-matrix.outputs.matrix }} -# matrix-oldproto: ${{ steps.set-matrix.outputs.matrix-oldproto }} -# -# docarray-test: -# needs: prep-testbed -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# python-version: [3.7] -# test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} -# steps: -# - uses: actions/checkout@v2.5.0 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Prepare enviroment -# run: | -# python -m pip install --upgrade pip -# python -m pip install wheel -# # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" -# pip install --no-cache-dir ".[test]" -# pip install --no-cache-dir ".[qdrant]" -# pip install --no-cache-dir ".[annlite]" -# pip install --no-cache-dir ".[weaviate]" -# pip install --no-cache-dir ".[elasticsearch]" -# pip install --no-cache-dir ".[redis]" -# pip install --no-cache-dir ".[full]" -# sudo apt-get install libsndfile1 -# - name: Test -# id: test -# run: | -# pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ -# -v -s -m "not gpu" ${{ matrix.test-path }} -# echo "codecov_flag=docarray" >> $GITHUB_OUTPUT -# timeout-minutes: 60 -# env: -# JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" -# - name: Check codecov file -# id: check_files -# uses: andstor/file-existence-action@v1 -# with: -# files: "coverage.xml" -# - name: Upload coverage from test to Codecov -# uses: codecov/codecov-action@v3.1.1 -# if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' -# with: -# file: coverage.xml -# flags: ${{ steps.test.outputs.codecov_flag }} -# fail_ci_if_error: false -# token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos -# -# docarray-oldproto-test: -# needs: prep-testbed -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# python-version: [3.7] -# test-path: ${{fromJson(needs.prep-testbed.outputs.matrix-oldproto)}} -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Prepare enviroment -# run: | -# python -m pip install --upgrade pip -# python -m pip install wheel -# # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" -# pip install --no-cache-dir ".[test]" -# pip install --no-cache-dir ".[qdrant]" -# pip install --no-cache-dir ".[annlite]" -# pip install --no-cache-dir ".[weaviate]" -# pip install --no-cache-dir ".[elasticsearch]" -# pip install --no-cache-dir ".[redis]" -# pip install --no-cache-dir ".[full]" -# pip install --no-cache-dir "protobuf<3.20.0" -# sudo apt-get install libsndfile1 -# - name: Test -# id: test -# run: | -# pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ -# -v -s -m "not gpu" ${{ matrix.test-path }} -# echo "::set-output name=codecov_flag::docarray" -# timeout-minutes: 60 -# env: -# JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" -# - name: Check codecov file -# id: check_files -# uses: andstor/file-existence-action@v1 -# with: -# files: "coverage.xml" -# - name: Upload coverage from test to Codecov -# uses: codecov/codecov-action@v2 -# if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' -# with: -# file: coverage.xml -# flags: ${{ steps.test.outputs.codecov_flag }} -# fail_ci_if_error: false -# token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos -# -# prerelease: -# needs: [docarray-test, docarray-oldproto-test] -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2.5.0 -# with: -# fetch-depth: 100 -# - name: Pre-release (.devN) -# run: | -# git fetch --depth=1 origin +refs/tags/*:refs/tags/* -# pip install twine wheel -# ./scripts/release.sh -# env: -# TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} -# TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} -# JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} + prep-testbed: + if: | + !startsWith(github.event.head_commit.message, 'chore') && + !startsWith(github.event.head_commit.message, 'build: hotfix') && + !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.5.0 + - id: set-matrix + run: | + sudo apt-get install jq + export value=$(bash scripts/get-all-test-paths.sh) + echo "matrix=$value" >> $GITHUB_OUTPUT + export value=$(bash scripts/get-all-oldproto-test-paths.sh) + echo "matrix-oldproto=$value" >> $GITHUB_OUTPUT + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + matrix-oldproto: ${{ steps.set-matrix.outputs.matrix-oldproto }} + + docarray-test: + needs: prep-testbed + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.7] + test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} + steps: + - uses: actions/checkout@v2.5.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Prepare enviroment + run: | + python -m pip install --upgrade pip + python -m pip install wheel + # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" + pip install --no-cache-dir ".[test]" + pip install --no-cache-dir ".[qdrant]" + pip install --no-cache-dir ".[annlite]" + pip install --no-cache-dir ".[weaviate]" + pip install --no-cache-dir ".[elasticsearch]" + pip install --no-cache-dir ".[redis]" + pip install --no-cache-dir ".[full]" + sudo apt-get install libsndfile1 + - name: Test + id: test + run: | + pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ + -v -s -m "not gpu" ${{ matrix.test-path }} + echo "codecov_flag=docarray" >> $GITHUB_OUTPUT + timeout-minutes: 60 + env: + JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" + - name: Check codecov file + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: "coverage.xml" + - name: Upload coverage from test to Codecov + uses: codecov/codecov-action@v3.1.1 + if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' + with: + file: coverage.xml + flags: ${{ steps.test.outputs.codecov_flag }} + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + + docarray-oldproto-test: + needs: prep-testbed + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.7] + test-path: ${{fromJson(needs.prep-testbed.outputs.matrix-oldproto)}} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Prepare enviroment + run: | + python -m pip install --upgrade pip + python -m pip install wheel + # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" + pip install --no-cache-dir ".[test]" + pip install --no-cache-dir ".[qdrant]" + pip install --no-cache-dir ".[annlite]" + pip install --no-cache-dir ".[weaviate]" + pip install --no-cache-dir ".[elasticsearch]" + pip install --no-cache-dir ".[redis]" + pip install --no-cache-dir ".[full]" + pip install --no-cache-dir "protobuf<3.20.0" + sudo apt-get install libsndfile1 + - name: Test + id: test + run: | + pytest --suppress-no-test-exit-code --cov=docarray --cov-report=xml \ + -v -s -m "not gpu" ${{ matrix.test-path }} + echo "::set-output name=codecov_flag::docarray" + timeout-minutes: 60 + env: + JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" + - name: Check codecov file + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: "coverage.xml" + - name: Upload coverage from test to Codecov + uses: codecov/codecov-action@v2 + if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' + with: + file: coverage.xml + flags: ${{ steps.test.outputs.codecov_flag }} + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + + prerelease: + needs: [docarray-test, docarray-oldproto-test] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.5.0 + with: + fetch-depth: 100 + - name: Pre-release (.devN) + run: | + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + pip install twine wheel + ./scripts/release.sh + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} check-readme-modification: runs-on: ubuntu-latest