From 06c7e0067cd0a2658feda656f4e107e7cf911e6e Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 14:24:20 +0200 Subject: [PATCH 1/8] chore: remove useless ci Signed-off-by: samsja --- .github/workflows/build_docs_on_branch.yml | 35 ---------------------- .github/workflows/ci.yml | 13 -------- 2 files changed, 48 deletions(-) delete mode 100644 .github/workflows/build_docs_on_branch.yml diff --git a/.github/workflows/build_docs_on_branch.yml b/.github/workflows/build_docs_on_branch.yml deleted file mode 100644 index 11238c4addb..00000000000 --- a/.github/workflows/build_docs_on_branch.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: BR -## this action is here only for the development phase of docarray v2 -## it will be removed once the new version is released -on: - push: - branches: - - feat-rewrite-v2 - - test-build-doc - -jobs: - - deploy-to-netlify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.5.0 - - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - uses: actions/setup-node@v2 - with: - node-version: '14' - - name: Build and Deploy - run: | - npm i -g netlify-cli - python -m pip install --upgrade pip - python -m pip install poetry - python -m poetry config virtualenvs.create false && python -m poetry install --no-interaction --no-ansi --all-extras - - cd docs - bash makedocs.sh - cd .. - netlify deploy --dir=site --alias="docarray-v2" --message="Deploying docs to docarray v2" - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN1 }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21dc58e0c76..0a36fdc0dcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,18 +79,6 @@ jobs: poetry install --all-extras poetry run mypy docarray -# prep-testbed: -# runs-on: ubuntu-latest -# needs: [lint-ruff, check-black, import-test] -# steps: -# - uses: actions/checkout@v2.5.0 -# - id: set-matrix -# run: | -# sudo apt-get install jq -# export value=$(bash scripts/get-all-test-paths.sh) -# echo "matrix=$value" >> $GITHUB_OUTPUT -# outputs: -# matrix: ${{ steps.set-matrix.outputs.matrix }} docarray-test: needs: [lint-ruff, check-black, import-test] @@ -99,7 +87,6 @@ jobs: fail-fast: false matrix: python-version: [3.7] -# test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} test-path: [tests/integrations, tests/units, tests/documentation] steps: - uses: actions/checkout@v2.5.0 From e2ca581930ed0e8538087ef99775a2d59d50bcee Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 14:34:47 +0200 Subject: [PATCH 2/8] chore: add workflow for release Signed-off-by: samsja --- .github/workflows/force-docs-build.yml | 91 ++++++++++++++++++++++ .github/workflows/force-release.yml | 56 +++++++++++++ .github/workflows/publish-docarray-org.yml | 62 +++++++++++++++ .github/workflows/tag.yml | 48 ++++++++++++ 4 files changed, 257 insertions(+) create mode 100644 .github/workflows/force-docs-build.yml create mode 100644 .github/workflows/force-release.yml create mode 100644 .github/workflows/publish-docarray-org.yml create mode 100644 .github/workflows/tag.yml diff --git a/.github/workflows/force-docs-build.yml b/.github/workflows/force-docs-build.yml new file mode 100644 index 00000000000..4bbe0b984eb --- /dev/null +++ b/.github/workflows/force-docs-build.yml @@ -0,0 +1,91 @@ +name: Manual Docs Build + +on: + workflow_dispatch: + inputs: + release_token: + description: 'Your release token' + required: true + triggered_by: + description: 'CD | TAG | MANUAL' + required: false + default: MANUAL + build_old_docs: + description: 'Whether to build old docs (TRUE | FALSE)' + type: string + default: 'FALSE' + package: + description: The name of the repo to build documentation for. + type: string + default: docarray + repo_owner: + description: The owner of the repo to build documentation for. Defaults to 'jina-ai'. + type: string + default: jina-ai + pages_branch: + description: Branch that Github Pages observes + type: string + default: gh-pages + git_config_name: + type: string + default: Jina Dev Bot + git_config_email: + type: string + default: dev-bot@jina.ai + +jobs: + token-check: + runs-on: ubuntu-latest + steps: + - name: Check release token + id: token-check + run: | + touch SUCCESS + if: inputs.release_token == env.release_token + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + - name: Fail release token + run: | + [[ -f SUCCESS ]] + + build-and-push-latest-docs: + needs: token-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry + python -m poetry config virtualenvs.create false && python -m poetry install --no-interaction --no-ansi --all-extras + + - name: docs Build + run: | + cd docs + bash makedocs.sh + cd .. + - name: Checkout to GH pages branch (${{ inputs.pages_branch }}) + run: | + git fetch origin ${{ inputs.pages_branch }}:${{ inputs.pages_branch }} --depth 1 + git checkout -f ${{ inputs.pages_branch }} + git reset --hard HEAD + - name: Moving old doc versions + run: | + cd docs + for i in $(cat _versions.json | jq '.[].version' | tr -d '"'); do if [ -d "$i" ]; then mv "$i" /tmp/gen-html; fi; done + - name: Swap in new docs + run: | + rm -rf ./docs + mv /tmp/gen-html ./docs + - name: Push it up! + run: | + git config --local user.email "${{ inputs.git_config_email }}" + git config --local user.name "${{ inputs.git_config_name }}" + git show --summary + git add ./docs && git commit -m "chore(docs): update docs due to ${{github.event_name}} on ${{github.repository}}" + git push origin ${{ inputs.pages_branch }} \ No newline at end of file diff --git a/.github/workflows/force-release.yml b/.github/workflows/force-release.yml new file mode 100644 index 00000000000..deb306b14d1 --- /dev/null +++ b/.github/workflows/force-release.yml @@ -0,0 +1,56 @@ +name: Manual Release + +on: + workflow_dispatch: + inputs: + release_token: + description: 'Your release token' + required: true + release_reason: + description: 'Short reason for this manual release' + required: true + +jobs: + token-check: + runs-on: ubuntu-latest + steps: + - name: Check release token + id: token-check + run: | + touch SUCCESS + if: inputs.release_token == env.release_token + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + - name: Fail release token + run: | + [[ -f SUCCESS ]] + + regular-release: + needs: token-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.5.0 + with: + token: ${{ secrets.JINA_DEV_BOT }} + fetch-depth: 100 # means max contribute history is limited to 100 lines +# submodules: true + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + - run: | + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + npm install git-release-notes + pip install twine wheel + ./scripts/release.sh final "${{ github.event.inputs.release_reason }}" "${{github.actor}}" + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} + - if: failure() + run: echo "nothing to release" + - name: bumping master version + uses: ad-m/github-push-action@v0.6.0 + with: + github_token: ${{ secrets.JINA_DEV_BOT }} + tags: true + branch: ${{ github.ref }} diff --git a/.github/workflows/publish-docarray-org.yml b/.github/workflows/publish-docarray-org.yml new file mode 100644 index 00000000000..7cab8afe911 --- /dev/null +++ b/.github/workflows/publish-docarray-org.yml @@ -0,0 +1,62 @@ +name: Manual Publish DocArray[dot]org + +on: + workflow_dispatch: + inputs: + release_token: + description: 'Your release token' + required: true + workflow_call: + secrets: + JINA_DEV_BOT: + required: true + +# uncomment this line for testing in PR +# push: + +jobs: + token-check: + runs-on: ubuntu-latest + steps: + - name: debug + run: | + echo ${{ github.event_name }} event triggered this action + - name: Check release token + id: token-check + run: | + touch SUCCESS + if: inputs.release_token == env.release_token + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + - name: Check trigger event + run: | + touch SUCCESS + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + - name: Fail release token + run: | + [[ -f SUCCESS ]] + + copy-readme: + needs: token-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + path: docarray + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + repository: docarray/docarray.github.io + path: docarray.github.io + token: ${{ secrets.JINA_DEV_BOT }} + - name: Check out DocArray page + run: | + cd ${GITHUB_WORKSPACE}/docarray.github.io + cp ${GITHUB_WORKSPACE}/docarray/README.md . + git config --local user.email "Jina Dev Bot" + git config --local user.name "dev-bot@jina.ai" + if [[ `git status --porcelain` ]]; then + git add README.md && git commit -s -m "chore(docs): sync up README from docarray" + git push + fi \ No newline at end of file diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml new file mode 100644 index 00000000000..cd658781be6 --- /dev/null +++ b/.github/workflows/tag.yml @@ -0,0 +1,48 @@ +name: Release CD + +on: + push: + tags: + - "v*" # push to version tags trigger the build + +#on: +# push: +# branches-ignore: +# - '**' # temporally disable this action + +jobs: + update-doc: + runs-on: ubuntu-latest + steps: + - uses: benc-uk/workflow-dispatch@v1 + with: + workflow: Manual Docs Build + token: ${{ secrets.JINA_DEV_BOT }} + inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "TAG", "build_old_docs": "TRUE"}' + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + + create-release: + needs: update-doc + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2.5.0 + with: + ref: 'main' + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + - run: | + python scripts/get-last-release-note.py + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: đź’« Patch ${{ github.ref }} + body_path: 'tmp.md' + draft: false + prerelease: false From 1f69c2e33a3decff226856a2d2ccfbc5a1645cd3 Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 14:41:28 +0200 Subject: [PATCH 3/8] chore: add release script Signed-off-by: samsja --- scripts/release.sh | 119 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 scripts/release.sh diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000000..48a5dd513e9 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +# Requirements +# brew install hub +# npm install -g git-release-notes +# pip install twine wheel + +set -ex + +INIT_FILE='docarray/__init__.py' +VER_TAG='__version__ = ' +RELEASENOTE='./node_modules/.bin/git-release-notes' + +function escape_slashes { + sed 's/\//\\\//g' +} + +function update_ver_line { + local OLD_LINE_PATTERN=$1 + local NEW_LINE=$2 + local FILE=$3 + + local NEW=$(echo "${NEW_LINE}" | escape_slashes) + sed -i '/'"${OLD_LINE_PATTERN}"'/s/.*/'"${NEW}"'/' "${FILE}" + head -n10 ${FILE} +} + + +function clean_build { + rm -rf dist + rm -rf *.egg-info + rm -rf build +} + +function pub_pypi { + # publish to pypi + clean_build + poetry publish --build + clean_build +} + +function git_commit { + git config --local user.email "dev-bot@jina.ai" + git config --local user.name "Jina Dev Bot" + git tag "v$RELEASE_VER" -m "$(cat ./CHANGELOG.tmp)" + git add $INIT_FILE ./CHANGELOG.md + git commit -m "chore(version): the next version will be $NEXT_VER" -m "build($RELEASE_ACTOR): $RELEASE_REASON" +} + + + +function make_release_note { + ${RELEASENOTE} ${LAST_VER}..HEAD .github/release-template.ejs > ./CHANGELOG.tmp + head -n10 ./CHANGELOG.tmp + printf '\n%s\n\n%s\n%s\n\n%s\n\n%s\n\n' "$(cat ./CHANGELOG.md)" "" "## Release Note (\`${RELEASE_VER}\`)" "> Release time: $(date +'%Y-%m-%d %H:%M:%S')" "$(cat ./CHANGELOG.tmp)" > ./CHANGELOG.md +} + +BRANCH=$(git rev-parse --abbrev-ref HEAD) + + +LAST_UPDATE=`git show --no-notes --format=format:"%H" $BRANCH | head -n 1` +LAST_COMMIT=`git show --no-notes --format=format:"%H" origin/$BRANCH | head -n 1` + +if [ $LAST_COMMIT != $LAST_UPDATE ]; then + printf "Your local $BRANCH is behind the remote master, exit\n" + exit 1; +fi + +# release the current version +export RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2) +LAST_VER=$(git tag -l | sort -V | tail -n1) +printf "last version: \e[1;32m$LAST_VER\e[0m\n" + +# Update new _versions.json if necessary +python ./scripts/prepend_version_json.py --version "v$RELEASE_VER" +git add ./docs/_versions.json + +if [[ $1 == "final" ]]; then + printf "this will be a final release: \e[1;33m$RELEASE_VER\e[0m\n" + + NEXT_VER=$(echo $RELEASE_VER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}') + printf "bump master version to: \e[1;32m$NEXT_VER\e[0m\n" + + make_release_note + + pub_pypi + + VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\' + update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE" + RELEASE_REASON="$2" + RELEASE_ACTOR="$3" + git_commit +elif [[ $1 == 'rc' ]]; then + printf "this will be a release candidate: \e[1;33m$RELEASE_VER\e[0m\n" + DOT_RELEASE_VER=$(echo $RELEASE_VER | sed "s/rc/\./") + NEXT_VER=$(echo $DOT_RELEASE_VER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}') + NEXT_VER=$(echo $NEXT_VER | sed "s/\.\([^.]*\)$/rc\1/") + printf "bump master version to: \e[1;32m$NEXT_VER\e[0m, this will be the next version\n" + + make_release_note + + pub_pypi + + VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\' + update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE" + RELEASE_REASON="$2" + RELEASE_ACTOR="$3" + git_commit +else + # as a prerelease, pypi update only, no back commit etc. + COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count) + NEXT_VER=$RELEASE_VER".dev"$COMMITS_SINCE_LAST_VER + printf "this will be a developmental release: \e[1;33m$NEXT_VER\e[0m\n" + + VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\' + update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE" + + pub_pypi +fi From 98a0ac8ddd437ab02da7237d6380c22f85c381bb Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 14:42:39 +0200 Subject: [PATCH 4/8] chore: add md files Signed-off-by: samsja --- GOVERNANCE.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 GOVERNANCE.md diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 00000000000..af08a0539fd --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,57 @@ +# Overview + +This project aims to be governed in a transparent, accessible way for the benefit of the community. All participation in this project is open and not bound to corporate affilation. Participants are bound to the project's [Code of Conduct](./.github/CODE_OF_CONDUCT.md). + +# Project roles + +## Contributor + +The contributor role is the starting role for anyone participating in the project and wishing to contribute code. + +### Process for becoming a contributor + +* Review the [Contribution Guidelines](./CONTRIBUTING.md) to ensure your contribution is inline with the project's coding and styling guidelines. +* Submit your code as a PR with the appropriate DCO signoff +* Have your submission approved by the committer(s) and merged into the codebase. + +## Core developers + +The core developers role enables the contributor to commit code directly to the repository, but also comes with the responsibility of being a responsible leader in the community. + +Currently core developers of DocArray are (in alphabetic order): [Alaeddine Abdessalem](https://github.com/alaeddine-13), [Charlotte Gerhaher](https://github.com/anna-charlotte), [Anne Yang](https://github.com/AnneYang720), [Han Xiao](https://github.com/hanxiao), [Joan Fontanals MartĂ­nez](https://github.com/JoanFM), [Johannes Messner](https://github.com/JohannesMessner),[Nan wang](https://github.com/nan-wang),[Sami Jaghouar](https://github.com/samsja). + + +### Process for becoming a core developer + +Contributors can become core developers by beeing nominated by at least one other core developers of DocArray. There will be a vote by the current core developers. While it is expected that most votes will be unanimous, a two-thirds majority of the cast votes is enough. + +If you want to become a core developers, reach out to docs.docarray.org. + +## TSC members + +The Techincal Steering Commite (TSC) members are core developers who have additional responsibilities to ensure the smooth running of the project. TSC members are expected to participate in strategic planning, and approve changes to the governance model. The purpose of the TSC is to ensure a smooth progress from the big-picture perspective. + +Currently TSC menbers of DocArray are (in alphabetic order): [Alaeddine Abdessalem](https://github.com/alaeddine-13), [Han Xiao](https://github.com/hanxiao), [Joan Fontanals MartĂ­nez](https://github.com/JoanFM), [Johannes Messner](https://github.com/JohannesMessner),[Nan wang](https://github.com/nan-wang),[Sami Jaghouar](https://github.com/samsja). + + +One of the TSC members is the chairperson of the TSC and should ensure the smooth running of the TSC. They do not have more voting power that other TSC menber. + +Currently [JoanFM](https://github.com/JoanFM) is the chairperson of the TSC + +### Process for becoming a TSC member + +Core developers can become TSC menbers by beeing nominated by at least one TSC member. There will be a vote by the current TSC menbers and the vote need to be unanimous for the nominated person to be accepted as a TSC menber. TSC members who do not actively engage with the TSC duties are expected to resign. + +# Release Process + +Project releases will occur on a scheduled basis as agreed by the committers. + +# Communication + +This project, just like all open source, is a global community. In addition to the [Code of Conduct](./.github/CODE_OF_CONDUCT.md), this project will: + +* Keep all communication on open channels ( mailing list, forums, chat ). +* Be respectful of time and language differences between community members ( such as scheduling meetings, email/issue responsiveness, etc ). +* Ensure tools are able to be used by community members regardless of their region. + +If you have concerns about communication challenges for this project, please contact the committers. From d10121535080b395181beea9435a0769c04716e3 Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 14:44:02 +0200 Subject: [PATCH 5/8] chore: add lf ai stuff Signed-off-by: samsja --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 1cee23cf2c0..2e56ff72485 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ DocArray handles your data while integrating seamlessly with the rest of your ** > - [Coming from FastAPI](#coming-from-fastapi) > - [Coming from a vector database](#coming-from-vector-database) +DocArray was released under the open-source [Apache License 2.0](https://github.com/docarray/docarray/blob/main/LICENSE) in January 2022. It is currently a sandbox project under [LF AI & Data Foundation](https://lfaidata.foundation/). ## Represent @@ -804,3 +805,5 @@ pip install "git+https://github.com/docarray/docarray@feat-rewrite-v2#egg=docarr - [Donation to Linux Foundation AI&Data blog post](https://jina.ai/news/donate-docarray-lf-for-inclusive-standard-multimodal-data-model/) - ["Legacy" DocArray github page](https://github.com/docarray/docarray) - ["Legacy" DocArray documentation](https://docarray.jina.ai/) + +> DocArray is a trademark of LF AI Projects, LLC \ No newline at end of file From 4f4d38487e38db2c7a9accabec973933471bca98 Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 15:06:33 +0200 Subject: [PATCH 6/8] chore: add v1 branch name Signed-off-by: samsja --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2e56ff72485..dc1c418dafc 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@

+> ⬆️ **DocArray v2**: This readme refer to the second version of DocArray (starting at 0.30). If you want to use the old +> DocArray v1 version (below 0.30) check out the [docarray-v1-fixe](https://github.com/docarray/docarray/tree/docarray-v1-fixes) branch + + DocArray is a library for **representing, sending and storing multi-modal data**, perfect for **Machine Learning applications**. Those are the three pillars of DocArray, and you can check them out individually: From 84913e77f75035aeb058ee3f7fc5962f17909f47 Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 15:43:01 +0200 Subject: [PATCH 7/8] chore: make licence a md file Signed-off-by: samsja --- LICENSE => LICENSE.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE => LICENSE.md (100%) diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md From 36ab9bf12a026b8b242c1e1420b0cadd45f01fd8 Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 14 Apr 2023 15:43:41 +0200 Subject: [PATCH 8/8] chore: bump versiom Signed-off-by: samsja --- docarray/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/__init__.py b/docarray/__init__.py index 2bffdc80803..6767bafd000 100644 --- a/docarray/__init__.py +++ b/docarray/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.30.0a3' +__version__ = '0.30.0' import logging