Skip to content

Commit 6255a5b

Browse files
haobibohaobibo
authored
CI update - publish to test on PR (#7)
* Update pip.yml * fix API handler multipart process
1 parent 736556f commit 6255a5b

11 files changed

Lines changed: 207 additions & 45 deletions

File tree

‎.gitattributes‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,17 @@
1-
*.py text eol=lf
2-
*.sh text eol=lf
3-
*.md text eol=lf
4-
*.ipynb text eol=lf
1+
# Ignore all differences in line endings
2+
*.md text eol=lf
3+
*.py text eol=lf
4+
*.sh text eol=lf
5+
*.conf text eol=lf
6+
*.yml text eol=lf
7+
*.ipynb text eol=lf
8+
Dockerfile* text eol=lf
9+
10+
*.tgz filter=lfs diff=lfs merge=lfs -text
11+
*.h5 filter=lfs diff=lfs merge=lfs -text
12+
*.jsonl filter=lfs diff=lfs merge=lfs -text
13+
*.bin filter=lfs diff=lfs merge=lfs -text
14+
*.png filter=lfs diff=lfs merge=lfs -text
15+
*.jpg filter=lfs diff=lfs merge=lfs -text
16+
*.jpeg filter=lfs diff=lfs merge=lfs -text
17+
*.webp filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/pip.yml‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Secret Varaibles required in GitHub secrets: TWINE_USERNAME, TWINE_PASSWORD
1+
# Secret Variables required in GitHub secrets: TWINE_USERNAME, TWINE_PASSWORD / TWINE_USERNAME_TEST, TWINE_PASSWORD_TEST
22

33
name: CI - PYPI
44

@@ -7,8 +7,10 @@ on:
77
# Triggers the workflow on push or pull request events but only for the main branch
88
push:
99
branches: [ main ]
10+
paths-ignore: [ "*.md" ]
1011
pull_request:
1112
branches: [ main ]
13+
paths-ignore: [ "*.md" ]
1214

1315
# Allows you to run this workflow manually from the Actions tab
1416
workflow_dispatch:
@@ -38,13 +40,16 @@ jobs:
3840
env:
3941
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
4042
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
43+
TWINE_USERNAME_TEST: ${{ secrets.TWINE_USERNAME_TEST }}
44+
TWINE_PASSWORD_TEST: ${{ secrets.TWINE_PASSWORD_TEST }}
4145
run: |
42-
cd src && ls -alh
46+
env | sort -f && cd src && ls -alh
4347
sudo python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
4448
sudo python setup.py sdist bdist_wheel
4549
ls -alh ./dist
46-
if [ "$GITHUB_REPOSITORY" = "QPod/aloha" ] ; then
47-
twine upload dist/* --verbose -u "$TWINE_USERNAME" -p "$TWINE_PASSWORD" ;
48-
elif [ ! -z "$TWINE_USERNAME" ]; then
49-
twine upload dist/* --verbose -u "$TWINE_USERNAME" -p "$TWINE_PASSWORD" --repository-url "https://test.pypi.org/legacy/" ;
50+
if [ "${GITHUB_REPOSITORY}" = "QPod/aloha" ] && [ "${GITHUB_HEAD_REF}" = "main" ] ; then
51+
twine upload dist/* --verbose -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" ;
52+
elif [ ! -z "${TWINE_USERNAME_TEST}" ]; then
53+
twine upload dist/* --verbose -u "${TWINE_USERNAME_TEST}" -p "${TWINE_PASSWORD_TEST}" \
54+
--repository-url "https://test.pypi.org/legacy/" ;
5055
fi

‎rel/run_build.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#! /bin/sh
2+
set -ex
3+
4+
SCRIPT_PWD=$(dirname "$0")
5+
. "${SCRIPT_PWD}/run_common.sh"
6+
. "${SCRIPT_PWD}/run_config.sh"
7+
8+
docker_build "${IMAGE_NAME}" "${IMAGE_TAG}" tool/Dockerfile
9+
docker tag "${IMAGE_NAME}:${IMAGE_TAG}" "${IMAGE}"
10+
11+
docker_push "${IMAGE_NAME}"

‎rel/run_common.sh‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#! /bin/sh
2+
set -ex
3+
4+
docker_build() {
5+
IMAGE_NAME=$1
6+
IMAGE_TAG=$2
7+
DOCKERFILE=$3
8+
shift 3
9+
IMAGE_TAG=${IMAGE_TAG:-"latest"}
10+
TIMESTAMP=$(date +'%Y-%m%d-%H%M')
11+
DOCKERFILE=${DOCKERFILE:-"Dockerfile"}
12+
13+
# build the docker image and tag as timestamp
14+
docker build --rm -t "${IMAGE_NAME}:${IMAGE_TAG}" -f "${DOCKERFILE}" .
15+
docker tag "${IMAGE_NAME}:${IMAGE_TAG}" "${IMAGE_NAME}:${TIMESTAMP}"
16+
docker images | grep "${IMAGE_NAME}"
17+
}
18+
19+
docker_push() {
20+
IMAGE_NAME=$1
21+
shift 1
22+
echo "Pushing docker image: ${IMAGE_NAME}"
23+
while
24+
timeout -k 605s 600s bash -c "docker push ${IMAGE_NAME}"
25+
[ $? = 124 ]
26+
do
27+
ehco "Timeout pushing docker image, retrying..."
28+
sleep 2
29+
done
30+
}
31+
32+
docker_destroy() {
33+
CONTAINER_NAME=$1
34+
shift 1
35+
(docker stop "${CONTAINER_NAME}") || true
36+
(docker rm "${CONTAINER_NAME}") || true
37+
}
38+
39+
docker_pull() {
40+
IMAGE_NAME=$1
41+
IMAGE_TAG=$2
42+
shift 2
43+
IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
44+
TIMESTAMP=$(date +'%Y-%m%d-%H%M')
45+
46+
docker tag "${IMAGE}" "${IMAGE_NAME}:bak-${TIMESTAMP}" || true
47+
docker rmi "${IMAGE}" || true
48+
49+
echo "Pulling new docker image: ${IMAGE}"
50+
while
51+
timeout -k 185s 180s bash -c "docker pull ${IMAGE_NAME}"
52+
[ $? = 124 ]
53+
do
54+
echo "Timeout pulling docker image, retrying..."
55+
sleep 2
56+
done
57+
58+
docker images | grep "${IMAGE_NAME}"
59+
}
60+
61+
docker_compose_prepare() {
62+
python3 -m pip install -Uq \
63+
docker-compose pip
64+
65+
alias docker-compose='python3 -m compose '
66+
docker-compose version
67+
}
68+
69+
docker_compose_deploy() {
70+
ENVIRONMENT=$1
71+
shift
72+
echo "Deploy containers for environment ${ENVIRONMENT}"
73+
rm -rf ./config ./*.yml || true
74+
/bin/cp -r ./deploy/profile_${ENVIRONMENT}/* ./
75+
76+
if [ -f "traefik-docker-compose.yml" ]; then
77+
echo "Found traefik docker-compose file, restarting svc-traefik..."
78+
(docker-compose -f traefik-docker-compose.yml down --remove-orphans || true) && docker-compose -f traefik-docker-compose.yml up -d
79+
else
80+
echo "Ignore traefik svc since no `traefik-docker-compose.yml` file found..."
81+
fi
82+
(docker-compose down || true) && docker-compose up -d
83+
84+
docker ps -a
85+
86+
echo "Removing old docker images..."
87+
docker images | grep 'days ago\|weeks ago\|months ago\|years ago' | awk '{print $1":"$2}' | xargs docker rmi || true
88+
}

‎rel/run_config.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#! /bin/sh
2+
set -ex
3+
4+
export PREFIX=${DOCKER_PREFIX:-"docker.io/"}
5+
export NAMESPACE="qpod"
6+
export IMAGE="aloha-app"
7+
8+
export IMAGE_TAG="latest"
9+
export IMAGE_NAME="${PREFIX}${NAMESPACE}/${IMAGE}"

‎rel/run_deploy.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#! /bin/sh
2+
set -ex
3+
4+
SCRIPT_PWD=$(dirname "$0")
5+
. "$SCRIPT_PWD/run_common.sh"
6+
. "$SCRIPT_PWD/run_config.sh"
7+
8+
export ENVIRONMENT=$1
9+
shift 1
10+
echo "Run deploy job for environment ${ENVIRONMENT}"
11+
12+
echo '*****' | docker login --username='*****' ${PREFIX} --password-stdin
13+
docker_pull "${IMAGE_NAME}" "${IMAGE_TAG}"
14+
15+
docker_compose_prepare
16+
docker_compose_deploy "${ENVIRONMENT}"

‎rel/run_dev.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#! /bin/sh
2+
set -ex
3+
4+
docker-compose -f rel/deploy/profile_DEV/docker-compose.yml down || true
5+
docker-compose -f rel/deploy/profile_DEV/docker-compose.yml up --build -d
6+
7+
docker-compose -f rel/deploy/profile_DEV/traefik-docker-compose.yml down || true
8+
docker-compose -f rel/deploy/profile_DEV/traefik-docker-compose.yml up -d

‎script/run_dev.sh‎

Whitespace-only changes.

‎src/aloha/service/v0/api_handler.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,16 @@ def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]:
2727
pass
2828

2929
async def post(self, *args, **kwargs):
30-
try:
31-
req = self.request.body
32-
body = json.loads(req.decode('utf-8'))
33-
kwargs.update(body)
34-
except json.decoder.JSONDecodeError: # invalid request body, cannot be parsed as JSON
35-
return self.finish(_RESP_BAD_REQUEST)
30+
content_type: str = self.request.headers.get('Content-Type', 'application/json; charset=utf-8')
31+
if content_type.startswith('multipart/form-data'): # only parse files when 'Content-Type' starts with 'multipart/form-data'
32+
body_arguments = self.request.body_arguments
33+
else:
34+
try:
35+
body = self.request.body.decode('utf-8')
36+
body_arguments = json.loads(body)
37+
except (UnicodeDecodeError, json.decoder.JSONDecodeError): # invalid request body, cannot be parsed as JSON
38+
return self.finish(_RESP_BAD_REQUEST)
39+
kwargs.update(body_arguments)
3640

3741
resp = dict(code=5200, message=['success'])
3842
try:
@@ -41,7 +45,7 @@ async def post(self, *args, **kwargs):
4145
except Exception as e:
4246
if LOG.level == logging.DEBUG:
4347
self.LOG.error(e, exc_info=True)
44-
return self.finish({'code': 5201, 'message': [str(e)]})
48+
return self.finish({'code': 5201, 'message': [repr(e)]})
4549

4650
resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':'))
4751
return self.finish(resp)

‎src/aloha/service/v1/api_handler.py‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,21 @@ def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]:
3030
pass
3131

3232
async def post(self):
33-
try:
34-
req = self.request.body
35-
body = json.loads(req.decode('utf-8'))
36-
except json.decoder.JSONDecodeError: # invalid request body, cannot be parsed as JSON
37-
return self.finish(_RESP_BAD_REQUEST)
33+
content_type: str = self.request.headers.get('Content-Type', 'application/json; charset=utf-8')
34+
if content_type.startswith('multipart/form-data'): # only parse files when 'Content-Type' starts with 'multipart/form-data'
35+
body_arguments = self.request.body_arguments
36+
else:
37+
try:
38+
body = self.request.body.decode('utf-8')
39+
body_arguments = json.loads(body)
40+
except (UnicodeDecodeError, json.decoder.JSONDecodeError): # invalid request body, cannot be parsed as JSON
41+
return self.finish(_RESP_BAD_REQUEST)
3842

3943
try:
40-
salt_uuid = body.pop('salt_uuid')
41-
app_id = body.pop('app_id')
42-
sign = body.pop('sign')
43-
data = body.pop('data')
44+
salt_uuid = body_arguments.pop('salt_uuid')
45+
app_id = body_arguments.pop('app_id')
46+
sign = body_arguments.pop('sign')
47+
data = body_arguments.pop('data')
4448
except KeyError: # cannot find default key from parsed body
4549
return self.finish(_RESP_MISSING_ARGS)
4650

@@ -56,7 +60,7 @@ async def post(self):
5660
except Exception as e:
5761
if LOG.level == logging.DEBUG:
5862
self.LOG.error(e, exc_info=True)
59-
return self.finish({'code': 5201, 'message': [str(e)]})
63+
return self.finish({'code': 5201, 'message': [repr(e)]})
6064

6165
resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':'))
6266
return self.finish(resp)

0 commit comments

Comments
 (0)