diff --git a/badge_server/main.py b/badge_server/main.py index 650b2331..45bdd6cf 100644 --- a/badge_server/main.py +++ b/badge_server/main.py @@ -39,7 +39,8 @@ def _get_result_from_cache( package_name: str, - badge_type: badge_utils.BadgeType) -> dict: + badge_type: badge_utils.BadgeType, + commit_number: str = None) -> dict: """Get check result from cache.""" # Return unknown if package not in whitelist if not utils._is_package_in_whitelist([package_name]): @@ -49,7 +50,9 @@ def _get_result_from_cache( details={}) # Get the result from cache, return None if not in cache else: - result = cache.get('{}_{}'.format(package_name, badge_type.value)) + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name + result = cache.get('{}_{}'.format(package_key, badge_type.value)) if result is None: result = badge_utils._build_default_result( @@ -109,7 +112,7 @@ def _get_pair_status_for_packages(pkg_sets): return version_and_res -def _get_all_results_from_cache(package_name): +def _get_all_results_from_cache(package_name, commit_number=None): """Get all the check results from cache. Rules: @@ -118,13 +121,16 @@ def _get_all_results_from_cache(package_name): """ self_compat_res = _get_result_from_cache( package_name=package_name, - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) + badge_type=badge_utils.BadgeType.SELF_COMP_BADGE, + commit_number=commit_number) google_compat_res = _get_result_from_cache( package_name=package_name, - badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE) + badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE, + commit_number=commit_number) dependency_res = _get_result_from_cache( package_name=package_name, - badge_type=badge_utils.BadgeType.DEP_BADGE) + badge_type=badge_utils.BadgeType.DEP_BADGE, + commit_number=commit_number) if self_compat_res['py3']['status'] == 'SUCCESS' and \ google_compat_res['py3']['status'] == 'SUCCESS' and \ @@ -177,6 +183,8 @@ def one_badge_image(): badge_name = badge_utils.GITHUB_HEAD_NAME is_github = True + commit_number = badge_utils._calculate_commit_number(package_name) + force_run_check = flask.request.args.get('force_run_check') # Remove the last '/' from the url root url_prefix = flask.request.url_root[:-1] @@ -186,19 +194,23 @@ def one_badge_image(): requests.get(url_prefix + flask.url_for( 'self_compatibility_badge_image', package=package_name, - force_run_check=force_run_check)) + force_run_check=force_run_check, + commit_number=commit_number)) # Google compatibility badge requests.get(url_prefix + flask.url_for( 'google_compatibility_badge_image', package=package_name, - force_run_check=force_run_check)) + force_run_check=force_run_check, + commit_number=commit_number)) # Self dependency badge requests.get(url_prefix + flask.url_for( 'self_dependency_badge_image', package=package_name, - force_run_check=force_run_check)) + force_run_check=force_run_check, + commit_number=commit_number)) - status, timestamp, _, _, _ = _get_all_results_from_cache(package_name) + status, timestamp, _, _, _ = _get_all_results_from_cache( + package_name, commit_number=commit_number) color = badge_utils.STATUS_COLOR_MAPPING[status] details_link = url_prefix + flask.url_for('one_badge_target', @@ -224,15 +236,18 @@ def one_badge_image(): @app.route('/one_badge_target') def one_badge_target(): package_name = flask.request.args.get('package') + commit_number = badge_utils._calculate_commit_number(package_name) + status, _, self_compat_res, google_compat_res, dependency_res = \ - _get_all_results_from_cache(package_name) + _get_all_results_from_cache(package_name, commit_number) return flask.render_template( 'one-badge.html', package_name=package_name, self_compat_res=self_compat_res, google_compat_res=google_compat_res, - dependency_res=dependency_res) + dependency_res=dependency_res, + commit_number=commit_number) @app.route('/self_compatibility_badge_image') @@ -240,8 +255,11 @@ def self_compatibility_badge_image(): """Badge showing whether a package is compatible with itself.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') + commit_number = flask.request.args.get('commit_number') badge_name = flask.request.args.get('badge_name') + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'self compatibility' @@ -285,8 +303,7 @@ def run_check(): badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore - cache.set( - '{}_self_comp_badge'.format(package_name), version_and_res) + cache.set('{}_self_comp_badge'.format(package_key), version_and_res) if not utils._is_package_in_whitelist([package_name]): self_comp_res = badge_utils._build_default_result( @@ -294,7 +311,7 @@ def run_check(): status='UNKNOWN', details=badge_utils.PACKAGE_NOT_SUPPORTED) else: - self_comp_res = cache.get('{}_self_comp_badge'.format(package_name)) + self_comp_res = cache.get('{}_self_comp_badge'.format(package_key)) if self_comp_res is None: details = version_and_res @@ -347,7 +364,11 @@ def self_dependency_badge_image(): package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') + commit_number = flask.request.args.get('commit_number') + badge_name = flask.request.args.get('badge_name') + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'dependency status' @@ -382,8 +403,7 @@ def run_check(): badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore - cache.set( - '{}_dependency_badge'.format(package_name), res) + cache.set('{}_dependency_badge'.format(package_key), res) if not utils._is_package_in_whitelist([package_name]): dependency_res = badge_utils._build_default_result( @@ -391,8 +411,7 @@ def run_check(): status='UNKNOWN', details={}) else: - dependency_res = cache.get( - '{}_dependency_badge'.format(package_name)) + dependency_res = cache.get('{}_dependency_badge'.format(package_key)) if dependency_res is None: details = badge_utils.DEFAULT_DEPENDENCY_RESULT @@ -433,7 +452,11 @@ def google_compatibility_badge_image(): to one of the failure types, details can be found at the target link.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') + commit_number = flask.request.args.get('commit_number') + badge_name = flask.request.args.get('badge_name') + package_key = '{}_{}'.format( + package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'google compatibility' @@ -492,11 +515,9 @@ def run_check(): result = version_and_res # Write the result to Cloud Datastore - cache.set( - '{}_google_comp_badge'.format(package_name), result) + cache.set('{}_google_comp_badge'.format(package_key), result) - google_comp_res = cache.get( - '{}_google_comp_badge'.format(package_name)) + google_comp_res = cache.get('{}_google_comp_badge'.format(package_key)) if not utils._is_package_in_whitelist([package_name]): google_comp_res = badge_utils._build_default_result( diff --git a/badge_server/templates/one-badge.html b/badge_server/templates/one-badge.html index 3428f534..79a0de04 100644 --- a/badge_server/templates/one-badge.html +++ b/badge_server/templates/one-badge.html @@ -30,6 +30,10 @@

Package name: {{ package_name }}

+ {% if commit_number | length > 0 %} +

Commit number: {{ commit_number }} +

+ {% endif %}
diff --git a/badge_server/utils.py b/badge_server/utils.py index f322cb3d..e4faf891 100644 --- a/badge_server/utils.py +++ b/badge_server/utils.py @@ -15,7 +15,12 @@ """Common utils methods for badge server.""" import enum +import json +import logging import os +from urllib.parse import urlparse +from urllib.request import urlopen + from typing import Optional import pybadges @@ -39,6 +44,7 @@ URL_PREFIX = 'https://img.shields.io/badge/' GITHUB_HEAD_NAME = 'github head' +GITHUB_API = 'https://api.github.com/repos' SVG_CONTENT_TYPE = 'image/svg+xml' EMPTY_DETAILS = 'NO DETAILS' PACKAGE_NOT_SUPPORTED = "The package is not supported by checker server." @@ -141,3 +147,26 @@ def _get_badge(res: dict, badge_name: str) -> str: left_text=badge_name, right_text=status, right_color=color) + + +def _calculate_commit_number(package: str) -> Optional[str]: + """Calculate the github head version commit number.""" + url_parsed = urlparse(package) + if url_parsed.scheme and url_parsed.netloc == 'github.com': + try: + owner, repo, *_ = url_parsed.path[1:].split('/') + repo = repo.split('.git')[0] + except ValueError: + return None + else: + url = '{0}/{1}/{2}/commits'.format(GITHUB_API, owner, repo) + try: + with urlopen(url) as f: + commits = json.loads(f.read()) + return commits[0]['sha'] + except Exception as e: + logging.warning( + 'Unable to generate caching key for "%s": %s', package, e) + return None + + return None diff --git a/compatibility_lib/compatibility_lib/get_compatibility_data.py b/compatibility_lib/compatibility_lib/get_compatibility_data.py index 60699e18..88deaae0 100644 --- a/compatibility_lib/compatibility_lib/get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/get_compatibility_data.py @@ -53,25 +53,11 @@ def _result_dict_to_compatibility_result(results, python_version): return res_list -def _generate_pairs_for_github_head(): - """Generate pairs for each github head package with the PyPI packages. - - e.g. [(github_pkg, pkg1), (github_pkg, pkg2),...] - """ - pkg_pairs = [] - - for gh_pkg in configs.WHITELIST_URLS.keys(): - gh_pairs = [(gh_pkg, package) for package in configs.PKG_LIST] - pkg_pairs.extend(gh_pairs) - - return pkg_pairs - - def write_to_status_table(): - """Get the compatibility status for PyPI and github head versions.""" + """Get the compatibility status for PyPI versions.""" # Write self compatibility status to BigQuery self_res_list = [] - packages = configs.PKG_LIST + list(configs.WHITELIST_URLS.keys()) + packages = configs.PKG_LIST for py_version in [PY2, PY3]: results = checker.get_self_compatibility( python_version=py_version, @@ -88,14 +74,6 @@ def write_to_status_table(): res_list = _result_dict_to_compatibility_result(results, py_version) store.save_compatibility_statuses(res_list) - # For github head versions - pkg_sets = _generate_pairs_for_github_head() - results = checker.get_pairwise_compatibility( - python_version=py_version, - pkg_sets=pkg_sets) - res_list = _result_dict_to_compatibility_result(results, py_version) - store.save_compatibility_statuses(res_list) - if __name__ == '__main__': write_to_status_table() diff --git a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py index ead9ea48..ce5647c3 100644 --- a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py @@ -72,30 +72,6 @@ def mock_init(): 'compatibility_lib.get_compatibility_data.store', self.fake_store) - def test__generate_pairs_for_github_head(self): - pkg_list = ['package1', 'package2'] - gh_pkgs = { - 'gh_pkg1_url': 'gh_pkg1', - 'gh_pkg2_url':'gh_pkg2' - } - patch_pkg_list = mock.patch( - 'compatibility_lib.configs.PKG_LIST', pkg_list) - patch_gh_list = mock.patch( - 'compatibility_lib.configs.WHITELIST_URLS', gh_pkgs) - - with self.patch_constructor, self.patch_checker, self.patch_store,\ - patch_pkg_list, patch_gh_list: - from compatibility_lib import get_compatibility_data - - pairs = get_compatibility_data._generate_pairs_for_github_head() - - expected = [('gh_pkg1_url', 'package1'), - ('gh_pkg1_url', 'package2'), - ('gh_pkg2_url', 'package1'), - ('gh_pkg2_url', 'package2')] - - self.assertEqual(set(pairs), set(expected)) - def test__result_dict_to_compatibility_result(self): with self.patch_constructor, self.patch_checker, self.patch_store: from compatibility_lib import compatibility_store @@ -123,7 +99,7 @@ def test_write_to_status_table(self): saved_results = self.fake_store._packages_to_compatibility_result.get( frozenset({self.packages[0]})) self.assertIsNotNone(saved_results) - self.assertEqual(len(saved_results), 6) + self.assertEqual(len(saved_results), 4) saved_item = saved_results[0] self.assertEqual(saved_item.packages, self.packages) self.assertEqual(saved_item.dependency_info, self.dependency_info)