From f98ae014faf8b9fd202eb9e72906a7ef4965b6ff Mon Sep 17 00:00:00 2001 From: Daniel Vaz Gaspar Date: Sat, 5 Dec 2020 08:55:31 +0000 Subject: [PATCH] feat(releasing): support changelog csv export (#11893) * feat(releasing): support changelog csv export * fix lint and type annotations --- RELEASING/changelog.py | 106 ++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/RELEASING/changelog.py b/RELEASING/changelog.py index 4ddc96d029..f4435f182c 100644 --- a/RELEASING/changelog.py +++ b/RELEASING/changelog.py @@ -14,12 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# pylint: disable=no-value-for-parameter + +import csv as lib_csv import json import os import re +import sys from dataclasses import dataclass from time import sleep -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Iterator, List, Optional, Union from urllib import request from urllib.error import HTTPError @@ -37,6 +41,7 @@ class GitLog: time: str message: str pr_number: Union[int, None] = None + author_email: str = "" def __eq__(self, other: object) -> bool: """ A log entry is considered equal if it has the same PR number """ @@ -44,7 +49,7 @@ class GitLog: return other.pr_number == self.pr_number return False - def __repr__(self): + def __repr__(self) -> str: return f"[{self.pr_number}]: {self.message} {self.time} {self.author}" @@ -73,15 +78,16 @@ class GitChangeLog: sleep(self._wait) print() - def _fetch_github_rate_limit(self) -> Dict[str, Any]: + @staticmethod + def _fetch_github_rate_limit() -> Dict[str, Any]: """ Fetches current github rate limit info """ - with request.urlopen(f"https://api.github.com/rate_limit") as response: + with request.urlopen("https://api.github.com/rate_limit") as response: payload = json.loads(response.read()) return payload - def _fetch_github_pr(self, pr_number) -> Dict[str, Any]: + def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]: """ Fetches a github PR info """ @@ -89,7 +95,7 @@ class GitChangeLog: try: self._wait_github_rate_limit() with request.urlopen( - f"https://api.github.com/repos/apache/incubator-superset/pulls/" + "https://api.github.com/repos/apache/incubator-superset/pulls/" f"{pr_number}" ) as response: payload = json.loads(response.read()) @@ -105,19 +111,20 @@ class GitChangeLog: github_login = self._github_login_cache.get(author_name) if github_login: return github_login - pr_info = self._fetch_github_pr(git_log.pr_number) - if pr_info: - github_login = pr_info["user"]["login"] - else: - github_login = author_name + if git_log.pr_number: + pr_info = self._fetch_github_pr(git_log.pr_number) + if pr_info: + github_login = pr_info["user"]["login"] + else: + github_login = author_name # set cache self._github_login_cache[author_name] = github_login return github_login - def _get_changelog_version_head(self): + def _get_changelog_version_head(self) -> str: return f"### {self._version} ({self._logs[0].time})" - def __repr__(self): + def __repr__(self) -> str: result = f"\n{self._get_changelog_version_head()}\n" for i, log in enumerate(self._logs): github_login = self._get_github_login(log) @@ -131,6 +138,19 @@ class GitChangeLog: print(f"\r {i}/{len(self._logs)}", end="", flush=True) return result + def __iter__(self) -> Iterator[Dict[str, Any]]: + for log in self._logs: + yield { + "pr_number": log.pr_number, + "pr_link": f"https://github.com/apache/incubator-superset/pull/" + f"{log.pr_number}", + "message": log.message, + "time": log.time, + "author": log.author, + "email": log.author_email, + "sha": log.sha, + } + class GitLogs: """ @@ -151,35 +171,36 @@ class GitLogs: def logs(self) -> List[GitLog]: return self._logs - def fetch(self): + def fetch(self) -> None: self._logs = list(map(self._parse_log, self._git_logs()))[::-1] def diff(self, git_logs: "GitLogs") -> List[GitLog]: return [log for log in git_logs.logs if log not in self._logs] - def __repr__(self): + def __repr__(self) -> str: return f"{self._git_ref}, Log count:{len(self._logs)}" - def _git_get_current_head(self) -> str: + @staticmethod + def _git_get_current_head() -> str: output = os.popen("git status | head -1").read() match = re.match("(?:HEAD detached at|On branch) (.*)", output) if not match: return "" return match.group(1) - def _git_checkout(self, git_ref: str): + def _git_checkout(self, git_ref: str) -> None: os.popen(f"git checkout {git_ref}").read() current_head = self._git_get_current_head() if current_head != git_ref: print(f"Could not checkout {git_ref}") - exit(1) + sys.exit(1) def _git_logs(self) -> List[str]: # let's get current git ref so we can revert it back current_git_ref = self._git_get_current_head() self._git_checkout(self._git_ref) output = ( - os.popen('git --no-pager log --pretty=format:"%h|%an|%ad|%s|"') + os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"') .read() .split("\n") ) @@ -187,18 +208,20 @@ class GitLogs: self._git_checkout(current_git_ref) return output - def _parse_log(self, log_item: str) -> GitLog: + @staticmethod + def _parse_log(log_item: str) -> GitLog: pr_number = None split_log_item = log_item.split("|") # parse the PR number from the log message - match = re.match(".*\(\#(\d*)\)", split_log_item[3]) + match = re.match(r".*\(\#(\d*)\)", split_log_item[4]) if match: pr_number = int(match.group(1)) return GitLog( sha=split_log_item[0], author=split_log_item[1], - time=split_log_item[2], - message=split_log_item[3], + author_email=split_log_item[2], + time=split_log_item[3], + message=split_log_item[4], pr_number=pr_number, ) @@ -217,13 +240,9 @@ def print_title(message: str) -> None: @click.group() @click.pass_context -@click.option( - "--previous_version", help="The previous release version", -) -@click.option( - "--current_version", help="The current release version", -) -def cli(ctx, previous_version: str, current_version: str): +@click.option("--previous_version", help="The previous release version", required=True) +@click.option("--current_version", help="The current release version", required=True) +def cli(ctx, previous_version: str, current_version: str) -> None: """ Welcome to change log generator """ previous_logs = GitLogs(previous_version) current_logs = GitLogs(current_version) @@ -235,7 +254,7 @@ def cli(ctx, previous_version: str, current_version: str): @cli.command("compare") @click.pass_obj -def compare(base_parameters): +def compare(base_parameters: BaseParameters) -> None: """ Compares both versions (by PR) """ previous_logs = base_parameters.previous_logs current_logs = base_parameters.current_logs @@ -255,14 +274,33 @@ def compare(base_parameters): @cli.command("changelog") +@click.option( + "--csv", help="The csv filename to export the changelog to", +) @click.pass_obj -def changelog(base_parameters): +def change_log(base_parameters: BaseParameters, csv: str) -> None: """ Outputs a changelog (by PR) """ previous_logs = base_parameters.previous_logs current_logs = base_parameters.current_logs previous_diff_logs = previous_logs.diff(current_logs) - print("Fetching github usernames, this may take a while:") - print(GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])) + logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1]) + if csv: + with open(csv, "w") as csv_file: + log_items = list(logs) + field_names = log_items[0].keys() + writer = lib_csv.DictWriter( + csv_file, + delimiter=",", + quotechar='"', + quoting=lib_csv.QUOTE_ALL, + fieldnames=field_names, + ) + writer.writeheader() + for log in logs: + writer.writerow(log) + else: + print("Fetching github usernames, this may take a while:") + print(logs) cli()