Skip to content
This repository was archived by the owner on Oct 21, 2022. It is now read-only.

Cache the github head version result based on commit number #201

Merged
liyanhui1228 merged 4 commits into from
Dec 14, 2018
Merged

Cache the github head version result based on commit number #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
066e53c
Cache the github head version result based on commit number
liyanhui1228 Dec 12, 2018
8c65376
fix tests
liyanhui1228 Dec 14, 2018
fec89ee
Display commit number on github head badge page
liyanhui1228 Dec 14, 2018
52a5f04
Merge branch 'master' into gh_cache
liyanhui1228 Dec 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 44 additions & 23 deletions badge_server/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@

def _get_result_from_cache(
package_name: str,
badge_type: badge_utils.BadgeType) -> dict:
badge_type: badge_utils.BadgeType,
commit_number: str = None) -> dict:
"""Get check result from cache."""
# Return unknown if package not in whitelist
if not utils._is_package_in_whitelist([package_name]):
Expand All @@ -49,7 +50,9 @@ def _get_result_from_cache(
details={})
# Get the result from cache, return None if not in cache
else:
result = cache.get('{}_{}'.format(package_name, badge_type.value))
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
result = cache.get('{}_{}'.format(package_key, badge_type.value))

if result is None:
result = badge_utils._build_default_result(
Expand Down Expand Up @@ -109,7 +112,7 @@ def _get_pair_status_for_packages(pkg_sets):
return version_and_res


def _get_all_results_from_cache(package_name):
def _get_all_results_from_cache(package_name, commit_number=None):
"""Get all the check results from cache.

Rules:
Expand All @@ -118,13 +121,16 @@ def _get_all_results_from_cache(package_name):
"""
self_compat_res = _get_result_from_cache(
package_name=package_name,
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE)
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
commit_number=commit_number)
google_compat_res = _get_result_from_cache(
package_name=package_name,
badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE)
badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE,
commit_number=commit_number)
dependency_res = _get_result_from_cache(
package_name=package_name,
badge_type=badge_utils.BadgeType.DEP_BADGE)
badge_type=badge_utils.BadgeType.DEP_BADGE,
commit_number=commit_number)

if self_compat_res['py3']['status'] == 'SUCCESS' and \
google_compat_res['py3']['status'] == 'SUCCESS' and \
Expand Down Expand Up @@ -177,6 +183,8 @@ def one_badge_image():
badge_name = badge_utils.GITHUB_HEAD_NAME
is_github = True

commit_number = badge_utils._calculate_commit_number(package_name)

force_run_check = flask.request.args.get('force_run_check')
# Remove the last '/' from the url root
url_prefix = flask.request.url_root[:-1]
Expand All @@ -186,19 +194,23 @@ def one_badge_image():
requests.get(url_prefix + flask.url_for(
'self_compatibility_badge_image',
package=package_name,
force_run_check=force_run_check))
force_run_check=force_run_check,
commit_number=commit_number))
# Google compatibility badge
requests.get(url_prefix + flask.url_for(
'google_compatibility_badge_image',
package=package_name,
force_run_check=force_run_check))
force_run_check=force_run_check,
commit_number=commit_number))
# Self dependency badge
requests.get(url_prefix + flask.url_for(
'self_dependency_badge_image',
package=package_name,
force_run_check=force_run_check))
force_run_check=force_run_check,
commit_number=commit_number))

status, timestamp, _, _, _ = _get_all_results_from_cache(package_name)
status, timestamp, _, _, _ = _get_all_results_from_cache(
package_name, commit_number=commit_number)
color = badge_utils.STATUS_COLOR_MAPPING[status]

details_link = url_prefix + flask.url_for('one_badge_target',
Expand All @@ -224,24 +236,30 @@ def one_badge_image():
@app.route('/one_badge_target')
def one_badge_target():
package_name = flask.request.args.get('package')
commit_number = badge_utils._calculate_commit_number(package_name)

status, _, self_compat_res, google_compat_res, dependency_res = \
_get_all_results_from_cache(package_name)
_get_all_results_from_cache(package_name, commit_number)

return flask.render_template(
'one-badge.html',
package_name=package_name,
self_compat_res=self_compat_res,
google_compat_res=google_compat_res,
dependency_res=dependency_res)
dependency_res=dependency_res,
commit_number=commit_number)


@app.route('/self_compatibility_badge_image')
def self_compatibility_badge_image():
"""Badge showing whether a package is compatible with itself."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')

badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name

if badge_name is None:
badge_name = 'self compatibility'
Expand Down Expand Up @@ -285,16 +303,15 @@ def run_check():
badge_utils.TIMESTAMP_FORMAT)

# Write the result to Cloud Datastore
cache.set(
'{}_self_comp_badge'.format(package_name), version_and_res)
cache.set('{}_self_comp_badge'.format(package_key), version_and_res)

if not utils._is_package_in_whitelist([package_name]):
self_comp_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
status='UNKNOWN',
details=badge_utils.PACKAGE_NOT_SUPPORTED)
else:
self_comp_res = cache.get('{}_self_comp_badge'.format(package_name))
self_comp_res = cache.get('{}_self_comp_badge'.format(package_key))

if self_comp_res is None:
details = version_and_res
Expand Down Expand Up @@ -347,7 +364,11 @@ def self_dependency_badge_image():

package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')

badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name

if badge_name is None:
badge_name = 'dependency status'
Expand Down Expand Up @@ -382,17 +403,15 @@ def run_check():
badge_utils.TIMESTAMP_FORMAT)

# Write the result to Cloud Datastore
cache.set(
'{}_dependency_badge'.format(package_name), res)
cache.set('{}_dependency_badge'.format(package_key), res)

if not utils._is_package_in_whitelist([package_name]):
dependency_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.DEP_BADGE,
status='UNKNOWN',
details={})
else:
dependency_res = cache.get(
'{}_dependency_badge'.format(package_name))
dependency_res = cache.get('{}_dependency_badge'.format(package_key))

if dependency_res is None:
details = badge_utils.DEFAULT_DEPENDENCY_RESULT
Expand Down Expand Up @@ -433,7 +452,11 @@ def google_compatibility_badge_image():
to one of the failure types, details can be found at the target link."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')

badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name

if badge_name is None:
badge_name = 'google compatibility'
Expand Down Expand Up @@ -492,11 +515,9 @@ def run_check():
result = version_and_res

# Write the result to Cloud Datastore
cache.set(
'{}_google_comp_badge'.format(package_name), result)
cache.set('{}_google_comp_badge'.format(package_key), result)

google_comp_res = cache.get(
'{}_google_comp_badge'.format(package_name))
google_comp_res = cache.get('{}_google_comp_badge'.format(package_key))

if not utils._is_package_in_whitelist([package_name]):
google_comp_res = badge_utils._build_default_result(
Expand Down
4 changes: 4 additions & 0 deletions badge_server/templates/one-badge.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@
<div class="col-md-12">
<h1 class="title-4">Package name: {{ package_name }}
</h1>
{% if commit_number | length > 0 %}
<h1 class="title-4">Commit number: {{ commit_number }}
</h1>
{% endif %}
<hr class="line-seprate">
</div>
</div>
Expand Down
29 changes: 29 additions & 0 deletions badge_server/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@
"""Common utils methods for badge server."""

import enum
import json
import logging
import os
from urllib.parse import urlparse
from urllib.request import urlopen

from typing import Optional

import pybadges
Expand All @@ -39,6 +44,7 @@

URL_PREFIX = 'https://img.shields.io/badge/'
GITHUB_HEAD_NAME = 'github head'
GITHUB_API = 'https://api.github.com/repos'
SVG_CONTENT_TYPE = 'image/svg+xml'
EMPTY_DETAILS = 'NO DETAILS'
PACKAGE_NOT_SUPPORTED = "The package is not supported by checker server."
Expand Down Expand Up @@ -141,3 +147,26 @@ def _get_badge(res: dict, badge_name: str) -> str:
left_text=badge_name,
right_text=status,
right_color=color)


def _calculate_commit_number(package: str) -> Optional[str]:
"""Calculate the github head version commit number."""
url_parsed = urlparse(package)
if url_parsed.scheme and url_parsed.netloc == 'github.com':
try:
owner, repo, *_ = url_parsed.path[1:].split('/')
repo = repo.split('.git')[0]
except ValueError:
return None
else:
url = '{0}/{1}/{2}/commits'.format(GITHUB_API, owner, repo)
try:
with urlopen(url) as f:
commits = json.loads(f.read())
return commits[0]['sha']
except Exception as e:
logging.warning(
'Unable to generate caching key for "%s": %s', package, e)
return None

return None
26 changes: 2 additions & 24 deletions compatibility_lib/compatibility_lib/get_compatibility_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,25 +53,11 @@ def _result_dict_to_compatibility_result(results, python_version):
return res_list


def _generate_pairs_for_github_head():
"""Generate pairs for each github head package with the PyPI packages.

e.g. [(github_pkg, pkg1), (github_pkg, pkg2),...]
"""
pkg_pairs = []

for gh_pkg in configs.WHITELIST_URLS.keys():
gh_pairs = [(gh_pkg, package) for package in configs.PKG_LIST]
pkg_pairs.extend(gh_pairs)

return pkg_pairs


def write_to_status_table():
"""Get the compatibility status for PyPI and github head versions."""
"""Get the compatibility status for PyPI versions."""
# Write self compatibility status to BigQuery
self_res_list = []
packages = configs.PKG_LIST + list(configs.WHITELIST_URLS.keys())
packages = configs.PKG_LIST
for py_version in [PY2, PY3]:
results = checker.get_self_compatibility(
python_version=py_version,
Expand All @@ -88,14 +74,6 @@ def write_to_status_table():
res_list = _result_dict_to_compatibility_result(results, py_version)
store.save_compatibility_statuses(res_list)

# For github head versions
pkg_sets = _generate_pairs_for_github_head()
results = checker.get_pairwise_compatibility(
python_version=py_version,
pkg_sets=pkg_sets)
res_list = _result_dict_to_compatibility_result(results, py_version)
store.save_compatibility_statuses(res_list)


if __name__ == '__main__':
write_to_status_table()
26 changes: 1 addition & 25 deletions compatibility_lib/compatibility_lib/test_get_compatibility_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,30 +72,6 @@ def mock_init():
'compatibility_lib.get_compatibility_data.store',
self.fake_store)

def test__generate_pairs_for_github_head(self):
pkg_list = ['package1', 'package2']
gh_pkgs = {
'gh_pkg1_url': 'gh_pkg1',
'gh_pkg2_url':'gh_pkg2'
}
patch_pkg_list = mock.patch(
'compatibility_lib.configs.PKG_LIST', pkg_list)
patch_gh_list = mock.patch(
'compatibility_lib.configs.WHITELIST_URLS', gh_pkgs)

with self.patch_constructor, self.patch_checker, self.patch_store,\
patch_pkg_list, patch_gh_list:
from compatibility_lib import get_compatibility_data

pairs = get_compatibility_data._generate_pairs_for_github_head()

expected = [('gh_pkg1_url', 'package1'),
('gh_pkg1_url', 'package2'),
('gh_pkg2_url', 'package1'),
('gh_pkg2_url', 'package2')]

self.assertEqual(set(pairs), set(expected))

def test__result_dict_to_compatibility_result(self):
with self.patch_constructor, self.patch_checker, self.patch_store:
from compatibility_lib import compatibility_store
Expand Down Expand Up @@ -123,7 +99,7 @@ def test_write_to_status_table(self):
saved_results = self.fake_store._packages_to_compatibility_result.get(
frozenset({self.packages[0]}))
self.assertIsNotNone(saved_results)
self.assertEqual(len(saved_results), 6)
self.assertEqual(len(saved_results), 4)
saved_item = saved_results[0]
self.assertEqual(saved_item.packages, self.packages)
self.assertEqual(saved_item.dependency_info, self.dependency_info)
Expand Down