From 066e53cf76c949164955b71872ef0bc241230ae9 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 11 Dec 2018 16:02:12 -0800 Subject: [PATCH 1/3] Cache the github head version result based on commit number --- badge_server/main.py | 64 ++++++++++++------- badge_server/utils.py | 31 +++++++++ .../get_compatibility_data.py | 26 +------- .../test_get_compatibility_data.py | 24 ------- compatibility_lib/compatibility_lib/utils.py | 14 ++++ 5 files changed, 89 insertions(+), 70 deletions(-) diff --git a/badge_server/main.py b/badge_server/main.py index 650b2331..7736a9f7 100644 --- a/badge_server/main.py +++ b/badge_server/main.py @@ -39,7 +39,8 @@ def _get_result_from_cache( package_name: str, - badge_type: badge_utils.BadgeType) -> dict: + badge_type: badge_utils.BadgeType, + commit_number: str=None) -> dict: """Get check result from cache.""" # Return unknown if package not in whitelist if not utils._is_package_in_whitelist([package_name]): @@ -49,7 +50,9 @@ def _get_result_from_cache( details={}) # Get the result from cache, return None if not in cache else: - result = cache.get('{}_{}'.format(package_name, badge_type.value)) + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name + result = cache.get('{}_{}'.format(package_key, badge_type.value)) if result is None: result = badge_utils._build_default_result( @@ -109,7 +112,7 @@ def _get_pair_status_for_packages(pkg_sets): return version_and_res -def _get_all_results_from_cache(package_name): +def _get_all_results_from_cache(package_name, commit_number): """Get all the check results from cache. Rules: @@ -118,13 +121,16 @@ def _get_all_results_from_cache(package_name): """ self_compat_res = _get_result_from_cache( package_name=package_name, - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) + badge_type=badge_utils.BadgeType.SELF_COMP_BADGE, + commit_number=commit_number) google_compat_res = _get_result_from_cache( package_name=package_name, - badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE) + badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE, + commit_number=commit_number) dependency_res = _get_result_from_cache( package_name=package_name, - badge_type=badge_utils.BadgeType.DEP_BADGE) + badge_type=badge_utils.BadgeType.DEP_BADGE, + commit_number=commit_number) if self_compat_res['py3']['status'] == 'SUCCESS' and \ google_compat_res['py3']['status'] == 'SUCCESS' and \ @@ -177,6 +183,8 @@ def one_badge_image(): badge_name = badge_utils.GITHUB_HEAD_NAME is_github = True + commit_number = badge_utils._calculate_commit_number(package_name) + force_run_check = flask.request.args.get('force_run_check') # Remove the last '/' from the url root url_prefix = flask.request.url_root[:-1] @@ -186,19 +194,23 @@ def one_badge_image(): requests.get(url_prefix + flask.url_for( 'self_compatibility_badge_image', package=package_name, - force_run_check=force_run_check)) + force_run_check=force_run_check, + commit_number=commit_number)) # Google compatibility badge requests.get(url_prefix + flask.url_for( 'google_compatibility_badge_image', package=package_name, - force_run_check=force_run_check)) + force_run_check=force_run_check, + commit_number=commit_number)) # Self dependency badge requests.get(url_prefix + flask.url_for( 'self_dependency_badge_image', package=package_name, - force_run_check=force_run_check)) + force_run_check=force_run_check, + commit_number=commit_number)) - status, timestamp, _, _, _ = _get_all_results_from_cache(package_name) + status, timestamp, _, _, _ = _get_all_results_from_cache( + package_name, commit_number=commit_number) color = badge_utils.STATUS_COLOR_MAPPING[status] details_link = url_prefix + flask.url_for('one_badge_target', @@ -224,8 +236,10 @@ def one_badge_image(): @app.route('/one_badge_target') def one_badge_target(): package_name = flask.request.args.get('package') + commit_number = badge_utils._calculate_commit_number(package_name) + status, _, self_compat_res, google_compat_res, dependency_res = \ - _get_all_results_from_cache(package_name) + _get_all_results_from_cache(package_name, commit_number) return flask.render_template( 'one-badge.html', @@ -240,8 +254,11 @@ def self_compatibility_badge_image(): """Badge showing whether a package is compatible with itself.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') + commit_number = flask.request.args.get('commit_number') badge_name = flask.request.args.get('badge_name') + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'self compatibility' @@ -285,8 +302,7 @@ def run_check(): badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore - cache.set( - '{}_self_comp_badge'.format(package_name), version_and_res) + cache.set('{}_self_comp_badge'.format(package_key), version_and_res) if not utils._is_package_in_whitelist([package_name]): self_comp_res = badge_utils._build_default_result( @@ -294,7 +310,7 @@ def run_check(): status='UNKNOWN', details=badge_utils.PACKAGE_NOT_SUPPORTED) else: - self_comp_res = cache.get('{}_self_comp_badge'.format(package_name)) + self_comp_res = cache.get('{}_self_comp_badge'.format(package_key)) if self_comp_res is None: details = version_and_res @@ -347,7 +363,11 @@ def self_dependency_badge_image(): package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') + commit_number = flask.request.args.get('commit_number') + badge_name = flask.request.args.get('badge_name') + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'dependency status' @@ -382,8 +402,7 @@ def run_check(): badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore - cache.set( - '{}_dependency_badge'.format(package_name), res) + cache.set('{}_dependency_badge'.format(package_key), res) if not utils._is_package_in_whitelist([package_name]): dependency_res = badge_utils._build_default_result( @@ -391,8 +410,7 @@ def run_check(): status='UNKNOWN', details={}) else: - dependency_res = cache.get( - '{}_dependency_badge'.format(package_name)) + dependency_res = cache.get('{}_dependency_badge'.format(package_key)) if dependency_res is None: details = badge_utils.DEFAULT_DEPENDENCY_RESULT @@ -433,7 +451,11 @@ def google_compatibility_badge_image(): to one of the failure types, details can be found at the target link.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') + commit_number = flask.request.args.get('commit_number') + badge_name = flask.request.args.get('badge_name') + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'google compatibility' @@ -492,11 +514,9 @@ def run_check(): result = version_and_res # Write the result to Cloud Datastore - cache.set( - '{}_google_comp_badge'.format(package_name), result) + cache.set('{}_google_comp_badge'.format(package_key), result) - google_comp_res = cache.get( - '{}_google_comp_badge'.format(package_name)) + google_comp_res = cache.get('{}_google_comp_badge'.format(package_key)) if not utils._is_package_in_whitelist([package_name]): google_comp_res = badge_utils._build_default_result( diff --git a/badge_server/utils.py b/badge_server/utils.py index f322cb3d..41e874f9 100644 --- a/badge_server/utils.py +++ b/badge_server/utils.py @@ -15,7 +15,12 @@ """Common utils methods for badge server.""" import enum +import json +import logging import os +from urllib.parse import urlparse +from urllib.request import urlopen + from typing import Optional import pybadges @@ -39,10 +44,13 @@ URL_PREFIX = 'https://img.shields.io/badge/' GITHUB_HEAD_NAME = 'github head' +GITHUB_API = 'https://api.github.com/repos' SVG_CONTENT_TYPE = 'image/svg+xml' EMPTY_DETAILS = 'NO DETAILS' PACKAGE_NOT_SUPPORTED = "The package is not supported by checker server." + + PY_VER_MAPPING = { 2: 'py2', 3: 'py3', @@ -141,3 +149,26 @@ def _get_badge(res: dict, badge_name: str) -> str: left_text=badge_name, right_text=status, right_color=color) + + +def _calculate_commit_number(package: str) -> Optional[str]: + """Calculate the github head version commit number.""" + url_parsed = urlparse(package) + if url_parsed.scheme and url_parsed.netloc == 'github.com': + try: + owner, repo, *_ = url_parsed.path[1:].split('/') + repo = repo.split('.git')[0] + except ValueError: + return None + else: + url = '{0}/{1}/{2}/commits'.format(GITHUB_API, owner, repo) + try: + with urlopen(url) as f: + commits = json.loads(f.read()) + return commits[0]['sha'] + except Exception as e: + logging.warning( + 'Unable to generate caching key for "%s": %s', package, e) + return None + + return None diff --git a/compatibility_lib/compatibility_lib/get_compatibility_data.py b/compatibility_lib/compatibility_lib/get_compatibility_data.py index 60699e18..88deaae0 100644 --- a/compatibility_lib/compatibility_lib/get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/get_compatibility_data.py @@ -53,25 +53,11 @@ def _result_dict_to_compatibility_result(results, python_version): return res_list -def _generate_pairs_for_github_head(): - """Generate pairs for each github head package with the PyPI packages. - - e.g. [(github_pkg, pkg1), (github_pkg, pkg2),...] - """ - pkg_pairs = [] - - for gh_pkg in configs.WHITELIST_URLS.keys(): - gh_pairs = [(gh_pkg, package) for package in configs.PKG_LIST] - pkg_pairs.extend(gh_pairs) - - return pkg_pairs - - def write_to_status_table(): - """Get the compatibility status for PyPI and github head versions.""" + """Get the compatibility status for PyPI versions.""" # Write self compatibility status to BigQuery self_res_list = [] - packages = configs.PKG_LIST + list(configs.WHITELIST_URLS.keys()) + packages = configs.PKG_LIST for py_version in [PY2, PY3]: results = checker.get_self_compatibility( python_version=py_version, @@ -88,14 +74,6 @@ def write_to_status_table(): res_list = _result_dict_to_compatibility_result(results, py_version) store.save_compatibility_statuses(res_list) - # For github head versions - pkg_sets = _generate_pairs_for_github_head() - results = checker.get_pairwise_compatibility( - python_version=py_version, - pkg_sets=pkg_sets) - res_list = _result_dict_to_compatibility_result(results, py_version) - store.save_compatibility_statuses(res_list) - if __name__ == '__main__': write_to_status_table() diff --git a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py index ead9ea48..3392f19e 100644 --- a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py @@ -72,30 +72,6 @@ def mock_init(): 'compatibility_lib.get_compatibility_data.store', self.fake_store) - def test__generate_pairs_for_github_head(self): - pkg_list = ['package1', 'package2'] - gh_pkgs = { - 'gh_pkg1_url': 'gh_pkg1', - 'gh_pkg2_url':'gh_pkg2' - } - patch_pkg_list = mock.patch( - 'compatibility_lib.configs.PKG_LIST', pkg_list) - patch_gh_list = mock.patch( - 'compatibility_lib.configs.WHITELIST_URLS', gh_pkgs) - - with self.patch_constructor, self.patch_checker, self.patch_store,\ - patch_pkg_list, patch_gh_list: - from compatibility_lib import get_compatibility_data - - pairs = get_compatibility_data._generate_pairs_for_github_head() - - expected = [('gh_pkg1_url', 'package1'), - ('gh_pkg1_url', 'package2'), - ('gh_pkg2_url', 'package1'), - ('gh_pkg2_url', 'package2')] - - self.assertEqual(set(pairs), set(expected)) - def test__result_dict_to_compatibility_result(self): with self.patch_constructor, self.patch_checker, self.patch_store: from compatibility_lib import compatibility_store diff --git a/compatibility_lib/compatibility_lib/utils.py b/compatibility_lib/compatibility_lib/utils.py index a69f1f3c..bfec1101 100644 --- a/compatibility_lib/compatibility_lib/utils.py +++ b/compatibility_lib/compatibility_lib/utils.py @@ -163,3 +163,17 @@ def _parse_datetime(date_string): date_string = date_string.replace('T', ' ') short_date = date_string.split(' ')[0] return datetime.strptime(short_date, DATETIME_FORMAT) + + +def _generate_pairs_for_github_head(): + """Generate pairs for each github head package with the PyPI packages. + + e.g. [(github_pkg, pkg1), (github_pkg, pkg2),...] + """ + pkg_pairs = [] + + for gh_pkg in configs.WHITELIST_URLS.keys(): + gh_pairs = [(gh_pkg, package) for package in configs.PKG_LIST] + pkg_pairs.extend(gh_pairs) + + return pkg_pairs From 8c65376dc78a6d564f463f9764fb30bf7ac354b4 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 13 Dec 2018 16:34:18 -0800 Subject: [PATCH 2/3] fix tests --- badge_server/main.py | 4 ++-- badge_server/utils.py | 2 -- .../compatibility_lib/test_get_compatibility_data.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/badge_server/main.py b/badge_server/main.py index 7736a9f7..ef9b3ebe 100644 --- a/badge_server/main.py +++ b/badge_server/main.py @@ -40,7 +40,7 @@ def _get_result_from_cache( package_name: str, badge_type: badge_utils.BadgeType, - commit_number: str=None) -> dict: + commit_number: str = None) -> dict: """Get check result from cache.""" # Return unknown if package not in whitelist if not utils._is_package_in_whitelist([package_name]): @@ -112,7 +112,7 @@ def _get_pair_status_for_packages(pkg_sets): return version_and_res -def _get_all_results_from_cache(package_name, commit_number): +def _get_all_results_from_cache(package_name, commit_number=None): """Get all the check results from cache. Rules: diff --git a/badge_server/utils.py b/badge_server/utils.py index 41e874f9..e4faf891 100644 --- a/badge_server/utils.py +++ b/badge_server/utils.py @@ -49,8 +49,6 @@ EMPTY_DETAILS = 'NO DETAILS' PACKAGE_NOT_SUPPORTED = "The package is not supported by checker server." - - PY_VER_MAPPING = { 2: 'py2', 3: 'py3', diff --git a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py index 3392f19e..ce5647c3 100644 --- a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py @@ -99,7 +99,7 @@ def test_write_to_status_table(self): saved_results = self.fake_store._packages_to_compatibility_result.get( frozenset({self.packages[0]})) self.assertIsNotNone(saved_results) - self.assertEqual(len(saved_results), 6) + self.assertEqual(len(saved_results), 4) saved_item = saved_results[0] self.assertEqual(saved_item.packages, self.packages) self.assertEqual(saved_item.dependency_info, self.dependency_info) From fec89ee4843ad104ab4f1568d67ce196c6acca4c Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 13 Dec 2018 17:12:44 -0800 Subject: [PATCH 3/3] Display commit number on github head badge page --- badge_server/main.py | 3 ++- badge_server/templates/one-badge.html | 4 ++++ compatibility_lib/compatibility_lib/utils.py | 14 -------------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/badge_server/main.py b/badge_server/main.py index ef9b3ebe..45bdd6cf 100644 --- a/badge_server/main.py +++ b/badge_server/main.py @@ -246,7 +246,8 @@ def one_badge_target(): package_name=package_name, self_compat_res=self_compat_res, google_compat_res=google_compat_res, - dependency_res=dependency_res) + dependency_res=dependency_res, + commit_number=commit_number) @app.route('/self_compatibility_badge_image') diff --git a/badge_server/templates/one-badge.html b/badge_server/templates/one-badge.html index 3428f534..79a0de04 100644 --- a/badge_server/templates/one-badge.html +++ b/badge_server/templates/one-badge.html @@ -30,6 +30,10 @@

Package name: {{ package_name }}

+ {% if commit_number | length > 0 %} +

Commit number: {{ commit_number }} +

+ {% endif %}
diff --git a/compatibility_lib/compatibility_lib/utils.py b/compatibility_lib/compatibility_lib/utils.py index bfec1101..a69f1f3c 100644 --- a/compatibility_lib/compatibility_lib/utils.py +++ b/compatibility_lib/compatibility_lib/utils.py @@ -163,17 +163,3 @@ def _parse_datetime(date_string): date_string = date_string.replace('T', ' ') short_date = date_string.split(' ')[0] return datetime.strptime(short_date, DATETIME_FORMAT) - - -def _generate_pairs_for_github_head(): - """Generate pairs for each github head package with the PyPI packages. - - e.g. [(github_pkg, pkg1), (github_pkg, pkg2),...] - """ - pkg_pairs = [] - - for gh_pkg in configs.WHITELIST_URLS.keys(): - gh_pairs = [(gh_pkg, package) for package in configs.PKG_LIST] - pkg_pairs.extend(gh_pairs) - - return pkg_pairs