diff --git a/.github/workflows/build_docs_on_branch.yml b/.github/workflows/build_docs_on_branch.yml deleted file mode 100644 index 11238c4addb..00000000000 --- a/.github/workflows/build_docs_on_branch.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: BR -## this action is here only for the development phase of docarray v2 -## it will be removed once the new version is released -on: - push: - branches: - - feat-rewrite-v2 - - test-build-doc - -jobs: - - deploy-to-netlify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.5.0 - - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - uses: actions/setup-node@v2 - with: - node-version: '14' - - name: Build and Deploy - run: | - npm i -g netlify-cli - python -m pip install --upgrade pip - python -m pip install poetry - python -m poetry config virtualenvs.create false && python -m poetry install --no-interaction --no-ansi --all-extras - - cd docs - bash makedocs.sh - cd .. - netlify deploy --dir=site --alias="docarray-v2" --message="Deploying docs to docarray v2" - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN1 }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21dc58e0c76..0a36fdc0dcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,18 +79,6 @@ jobs: poetry install --all-extras poetry run mypy docarray -# prep-testbed: -# runs-on: ubuntu-latest -# needs: [lint-ruff, check-black, import-test] -# steps: -# - uses: actions/checkout@v2.5.0 -# - id: set-matrix -# run: | -# sudo apt-get install jq -# export value=$(bash scripts/get-all-test-paths.sh) -# echo "matrix=$value" >> $GITHUB_OUTPUT -# outputs: -# matrix: ${{ steps.set-matrix.outputs.matrix }} docarray-test: needs: [lint-ruff, check-black, import-test] @@ -99,7 +87,6 @@ jobs: fail-fast: false matrix: python-version: [3.7] -# test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} test-path: [tests/integrations, tests/units, tests/documentation] steps: - uses: actions/checkout@v2.5.0 diff --git a/.github/workflows/force-docs-build.yml b/.github/workflows/force-docs-build.yml new file mode 100644 index 00000000000..4bbe0b984eb --- /dev/null +++ b/.github/workflows/force-docs-build.yml @@ -0,0 +1,91 @@ +name: Manual Docs Build + +on: + workflow_dispatch: + inputs: + release_token: + description: 'Your release token' + required: true + triggered_by: + description: 'CD | TAG | MANUAL' + required: false + default: MANUAL + build_old_docs: + description: 'Whether to build old docs (TRUE | FALSE)' + type: string + default: 'FALSE' + package: + description: The name of the repo to build documentation for. + type: string + default: docarray + repo_owner: + description: The owner of the repo to build documentation for. Defaults to 'jina-ai'. + type: string + default: jina-ai + pages_branch: + description: Branch that Github Pages observes + type: string + default: gh-pages + git_config_name: + type: string + default: Jina Dev Bot + git_config_email: + type: string + default: dev-bot@jina.ai + +jobs: + token-check: + runs-on: ubuntu-latest + steps: + - name: Check release token + id: token-check + run: | + touch SUCCESS + if: inputs.release_token == env.release_token + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + - name: Fail release token + run: | + [[ -f SUCCESS ]] + + build-and-push-latest-docs: + needs: token-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry + python -m poetry config virtualenvs.create false && python -m poetry install --no-interaction --no-ansi --all-extras + + - name: docs Build + run: | + cd docs + bash makedocs.sh + cd .. + - name: Checkout to GH pages branch (${{ inputs.pages_branch }}) + run: | + git fetch origin ${{ inputs.pages_branch }}:${{ inputs.pages_branch }} --depth 1 + git checkout -f ${{ inputs.pages_branch }} + git reset --hard HEAD + - name: Moving old doc versions + run: | + cd docs + for i in $(cat _versions.json | jq '.[].version' | tr -d '"'); do if [ -d "$i" ]; then mv "$i" /tmp/gen-html; fi; done + - name: Swap in new docs + run: | + rm -rf ./docs + mv /tmp/gen-html ./docs + - name: Push it up! + run: | + git config --local user.email "${{ inputs.git_config_email }}" + git config --local user.name "${{ inputs.git_config_name }}" + git show --summary + git add ./docs && git commit -m "chore(docs): update docs due to ${{github.event_name}} on ${{github.repository}}" + git push origin ${{ inputs.pages_branch }} \ No newline at end of file diff --git a/.github/workflows/force-release.yml b/.github/workflows/force-release.yml new file mode 100644 index 00000000000..deb306b14d1 --- /dev/null +++ b/.github/workflows/force-release.yml @@ -0,0 +1,56 @@ +name: Manual Release + +on: + workflow_dispatch: + inputs: + release_token: + description: 'Your release token' + required: true + release_reason: + description: 'Short reason for this manual release' + required: true + +jobs: + token-check: + runs-on: ubuntu-latest + steps: + - name: Check release token + id: token-check + run: | + touch SUCCESS + if: inputs.release_token == env.release_token + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + - name: Fail release token + run: | + [[ -f SUCCESS ]] + + regular-release: + needs: token-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.5.0 + with: + token: ${{ secrets.JINA_DEV_BOT }} + fetch-depth: 100 # means max contribute history is limited to 100 lines +# submodules: true + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + - run: | + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + npm install git-release-notes + pip install twine wheel + ./scripts/release.sh final "${{ github.event.inputs.release_reason }}" "${{github.actor}}" + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} + - if: failure() + run: echo "nothing to release" + - name: bumping master version + uses: ad-m/github-push-action@v0.6.0 + with: + github_token: ${{ secrets.JINA_DEV_BOT }} + tags: true + branch: ${{ github.ref }} diff --git a/.github/workflows/publish-docarray-org.yml b/.github/workflows/publish-docarray-org.yml new file mode 100644 index 00000000000..7cab8afe911 --- /dev/null +++ b/.github/workflows/publish-docarray-org.yml @@ -0,0 +1,62 @@ +name: Manual Publish DocArray[dot]org + +on: + workflow_dispatch: + inputs: + release_token: + description: 'Your release token' + required: true + workflow_call: + secrets: + JINA_DEV_BOT: + required: true + +# uncomment this line for testing in PR +# push: + +jobs: + token-check: + runs-on: ubuntu-latest + steps: + - name: debug + run: | + echo ${{ github.event_name }} event triggered this action + - name: Check release token + id: token-check + run: | + touch SUCCESS + if: inputs.release_token == env.release_token + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + - name: Check trigger event + run: | + touch SUCCESS + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + - name: Fail release token + run: | + [[ -f SUCCESS ]] + + copy-readme: + needs: token-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + path: docarray + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + repository: docarray/docarray.github.io + path: docarray.github.io + token: ${{ secrets.JINA_DEV_BOT }} + - name: Check out DocArray page + run: | + cd ${GITHUB_WORKSPACE}/docarray.github.io + cp ${GITHUB_WORKSPACE}/docarray/README.md . + git config --local user.email "Jina Dev Bot" + git config --local user.name "dev-bot@jina.ai" + if [[ `git status --porcelain` ]]; then + git add README.md && git commit -s -m "chore(docs): sync up README from docarray" + git push + fi \ No newline at end of file diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml new file mode 100644 index 00000000000..cd658781be6 --- /dev/null +++ b/.github/workflows/tag.yml @@ -0,0 +1,48 @@ +name: Release CD + +on: + push: + tags: + - "v*" # push to version tags trigger the build + +#on: +# push: +# branches-ignore: +# - '**' # temporally disable this action + +jobs: + update-doc: + runs-on: ubuntu-latest + steps: + - uses: benc-uk/workflow-dispatch@v1 + with: + workflow: Manual Docs Build + token: ${{ secrets.JINA_DEV_BOT }} + inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "TAG", "build_old_docs": "TRUE"}' + env: + release_token: ${{ secrets.DOCARRAY_RELEASE_TOKEN }} + + create-release: + needs: update-doc + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2.5.0 + with: + ref: 'main' + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + - run: | + python scripts/get-last-release-note.py + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: đź’« Patch ${{ github.ref }} + body_path: 'tmp.md' + draft: false + prerelease: false diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 00000000000..af08a0539fd --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,57 @@ +# Overview + +This project aims to be governed in a transparent, accessible way for the benefit of the community. All participation in this project is open and not bound to corporate affilation. Participants are bound to the project's [Code of Conduct](./.github/CODE_OF_CONDUCT.md). + +# Project roles + +## Contributor + +The contributor role is the starting role for anyone participating in the project and wishing to contribute code. + +### Process for becoming a contributor + +* Review the [Contribution Guidelines](./CONTRIBUTING.md) to ensure your contribution is inline with the project's coding and styling guidelines. +* Submit your code as a PR with the appropriate DCO signoff +* Have your submission approved by the committer(s) and merged into the codebase. + +## Core developers + +The core developers role enables the contributor to commit code directly to the repository, but also comes with the responsibility of being a responsible leader in the community. + +Currently core developers of DocArray are (in alphabetic order): [Alaeddine Abdessalem](https://github.com/alaeddine-13), [Charlotte Gerhaher](https://github.com/anna-charlotte), [Anne Yang](https://github.com/AnneYang720), [Han Xiao](https://github.com/hanxiao), [Joan Fontanals MartĂ­nez](https://github.com/JoanFM), [Johannes Messner](https://github.com/JohannesMessner),[Nan wang](https://github.com/nan-wang),[Sami Jaghouar](https://github.com/samsja). + + +### Process for becoming a core developer + +Contributors can become core developers by beeing nominated by at least one other core developers of DocArray. There will be a vote by the current core developers. While it is expected that most votes will be unanimous, a two-thirds majority of the cast votes is enough. + +If you want to become a core developers, reach out to docs.docarray.org. + +## TSC members + +The Techincal Steering Commite (TSC) members are core developers who have additional responsibilities to ensure the smooth running of the project. TSC members are expected to participate in strategic planning, and approve changes to the governance model. The purpose of the TSC is to ensure a smooth progress from the big-picture perspective. + +Currently TSC menbers of DocArray are (in alphabetic order): [Alaeddine Abdessalem](https://github.com/alaeddine-13), [Han Xiao](https://github.com/hanxiao), [Joan Fontanals MartĂ­nez](https://github.com/JoanFM), [Johannes Messner](https://github.com/JohannesMessner),[Nan wang](https://github.com/nan-wang),[Sami Jaghouar](https://github.com/samsja). + + +One of the TSC members is the chairperson of the TSC and should ensure the smooth running of the TSC. They do not have more voting power that other TSC menber. + +Currently [JoanFM](https://github.com/JoanFM) is the chairperson of the TSC + +### Process for becoming a TSC member + +Core developers can become TSC menbers by beeing nominated by at least one TSC member. There will be a vote by the current TSC menbers and the vote need to be unanimous for the nominated person to be accepted as a TSC menber. TSC members who do not actively engage with the TSC duties are expected to resign. + +# Release Process + +Project releases will occur on a scheduled basis as agreed by the committers. + +# Communication + +This project, just like all open source, is a global community. In addition to the [Code of Conduct](./.github/CODE_OF_CONDUCT.md), this project will: + +* Keep all communication on open channels ( mailing list, forums, chat ). +* Be respectful of time and language differences between community members ( such as scheduling meetings, email/issue responsiveness, etc ). +* Ensure tools are able to be used by community members regardless of their region. + +If you have concerns about communication challenges for this project, please contact the committers. diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md diff --git a/README.md b/README.md index 919c26a9652..26c7b4bc6bf 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@

+> ⬆️ **DocArray v2**: This readme refer to the second version of DocArray (starting at 0.30). If you want to use the old +> DocArray v1 version (below 0.30) check out the [docarray-v1-fixe](https://github.com/docarray/docarray/tree/docarray-v1-fixes) branch + + DocArray is a library for **representing, sending and storing multi-modal data**, perfect for **Machine Learning applications**. Those are the three pillars of DocArray, and you can check them out individually: @@ -35,6 +39,7 @@ DocArray handles your data while integrating seamlessly with the rest of your ** > - [Coming from FastAPI](#coming-from-fastapi) > - [Coming from a vector database](#coming-from-vector-database) +DocArray was released under the open-source [Apache License 2.0](https://github.com/docarray/docarray/blob/main/LICENSE) in January 2022. It is currently a sandbox project under [LF AI & Data Foundation](https://lfaidata.foundation/). ## Represent @@ -804,3 +809,5 @@ pip install "git+https://github.com/docarray/docarray@feat-rewrite-v2#egg=docarr - [Donation to Linux Foundation AI&Data blog post](https://jina.ai/news/donate-docarray-lf-for-inclusive-standard-multimodal-data-model/) - ["Legacy" DocArray github page](https://github.com/docarray/docarray) - ["Legacy" DocArray documentation](https://docarray.jina.ai/) + +> DocArray is a trademark of LF AI Projects, LLC \ No newline at end of file diff --git a/docarray/__init__.py b/docarray/__init__.py index 2bffdc80803..6767bafd000 100644 --- a/docarray/__init__.py +++ b/docarray/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.30.0a3' +__version__ = '0.30.0' import logging diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000000..48a5dd513e9 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +# Requirements +# brew install hub +# npm install -g git-release-notes +# pip install twine wheel + +set -ex + +INIT_FILE='docarray/__init__.py' +VER_TAG='__version__ = ' +RELEASENOTE='./node_modules/.bin/git-release-notes' + +function escape_slashes { + sed 's/\//\\\//g' +} + +function update_ver_line { + local OLD_LINE_PATTERN=$1 + local NEW_LINE=$2 + local FILE=$3 + + local NEW=$(echo "${NEW_LINE}" | escape_slashes) + sed -i '/'"${OLD_LINE_PATTERN}"'/s/.*/'"${NEW}"'/' "${FILE}" + head -n10 ${FILE} +} + + +function clean_build { + rm -rf dist + rm -rf *.egg-info + rm -rf build +} + +function pub_pypi { + # publish to pypi + clean_build + poetry publish --build + clean_build +} + +function git_commit { + git config --local user.email "dev-bot@jina.ai" + git config --local user.name "Jina Dev Bot" + git tag "v$RELEASE_VER" -m "$(cat ./CHANGELOG.tmp)" + git add $INIT_FILE ./CHANGELOG.md + git commit -m "chore(version): the next version will be $NEXT_VER" -m "build($RELEASE_ACTOR): $RELEASE_REASON" +} + + + +function make_release_note { + ${RELEASENOTE} ${LAST_VER}..HEAD .github/release-template.ejs > ./CHANGELOG.tmp + head -n10 ./CHANGELOG.tmp + printf '\n%s\n\n%s\n%s\n\n%s\n\n%s\n\n' "$(cat ./CHANGELOG.md)" "" "## Release Note (\`${RELEASE_VER}\`)" "> Release time: $(date +'%Y-%m-%d %H:%M:%S')" "$(cat ./CHANGELOG.tmp)" > ./CHANGELOG.md +} + +BRANCH=$(git rev-parse --abbrev-ref HEAD) + + +LAST_UPDATE=`git show --no-notes --format=format:"%H" $BRANCH | head -n 1` +LAST_COMMIT=`git show --no-notes --format=format:"%H" origin/$BRANCH | head -n 1` + +if [ $LAST_COMMIT != $LAST_UPDATE ]; then + printf "Your local $BRANCH is behind the remote master, exit\n" + exit 1; +fi + +# release the current version +export RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2) +LAST_VER=$(git tag -l | sort -V | tail -n1) +printf "last version: \e[1;32m$LAST_VER\e[0m\n" + +# Update new _versions.json if necessary +python ./scripts/prepend_version_json.py --version "v$RELEASE_VER" +git add ./docs/_versions.json + +if [[ $1 == "final" ]]; then + printf "this will be a final release: \e[1;33m$RELEASE_VER\e[0m\n" + + NEXT_VER=$(echo $RELEASE_VER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}') + printf "bump master version to: \e[1;32m$NEXT_VER\e[0m\n" + + make_release_note + + pub_pypi + + VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\' + update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE" + RELEASE_REASON="$2" + RELEASE_ACTOR="$3" + git_commit +elif [[ $1 == 'rc' ]]; then + printf "this will be a release candidate: \e[1;33m$RELEASE_VER\e[0m\n" + DOT_RELEASE_VER=$(echo $RELEASE_VER | sed "s/rc/\./") + NEXT_VER=$(echo $DOT_RELEASE_VER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}') + NEXT_VER=$(echo $NEXT_VER | sed "s/\.\([^.]*\)$/rc\1/") + printf "bump master version to: \e[1;32m$NEXT_VER\e[0m, this will be the next version\n" + + make_release_note + + pub_pypi + + VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\' + update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE" + RELEASE_REASON="$2" + RELEASE_ACTOR="$3" + git_commit +else + # as a prerelease, pypi update only, no back commit etc. + COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count) + NEXT_VER=$RELEASE_VER".dev"$COMMITS_SINCE_LAST_VER + printf "this will be a developmental release: \e[1;33m$NEXT_VER\e[0m\n" + + VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\' + update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE" + + pub_pypi +fi