feat(releasing): support changelog csv export (#11893)

* feat(releasing): support changelog csv export

* fix lint and type annotations
This commit is contained in:
Daniel Vaz Gaspar 2020-12-05 08:55:31 +00:00 committed by GitHub
parent 41738df77d
commit f98ae014fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 72 additions and 34 deletions

View File

@ -14,12 +14,16 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
# pylint: disable=no-value-for-parameter
import csv as lib_csv
import json import json
import os import os
import re import re
import sys
from dataclasses import dataclass from dataclasses import dataclass
from time import sleep from time import sleep
from typing import Any, Dict, List, Optional, Union from typing import Any, Dict, Iterator, List, Optional, Union
from urllib import request from urllib import request
from urllib.error import HTTPError from urllib.error import HTTPError
@ -37,6 +41,7 @@ class GitLog:
time: str time: str
message: str message: str
pr_number: Union[int, None] = None pr_number: Union[int, None] = None
author_email: str = ""
def __eq__(self, other: object) -> bool: def __eq__(self, other: object) -> bool:
""" A log entry is considered equal if it has the same PR number """ """ A log entry is considered equal if it has the same PR number """
@ -44,7 +49,7 @@ class GitLog:
return other.pr_number == self.pr_number return other.pr_number == self.pr_number
return False return False
def __repr__(self): def __repr__(self) -> str:
return f"[{self.pr_number}]: {self.message} {self.time} {self.author}" return f"[{self.pr_number}]: {self.message} {self.time} {self.author}"
@ -73,15 +78,16 @@ class GitChangeLog:
sleep(self._wait) sleep(self._wait)
print() print()
def _fetch_github_rate_limit(self) -> Dict[str, Any]: @staticmethod
def _fetch_github_rate_limit() -> Dict[str, Any]:
""" """
Fetches current github rate limit info Fetches current github rate limit info
""" """
with request.urlopen(f"https://api.github.com/rate_limit") as response: with request.urlopen("https://api.github.com/rate_limit") as response:
payload = json.loads(response.read()) payload = json.loads(response.read())
return payload return payload
def _fetch_github_pr(self, pr_number) -> Dict[str, Any]: def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]:
""" """
Fetches a github PR info Fetches a github PR info
""" """
@ -89,7 +95,7 @@ class GitChangeLog:
try: try:
self._wait_github_rate_limit() self._wait_github_rate_limit()
with request.urlopen( with request.urlopen(
f"https://api.github.com/repos/apache/incubator-superset/pulls/" "https://api.github.com/repos/apache/incubator-superset/pulls/"
f"{pr_number}" f"{pr_number}"
) as response: ) as response:
payload = json.loads(response.read()) payload = json.loads(response.read())
@ -105,19 +111,20 @@ class GitChangeLog:
github_login = self._github_login_cache.get(author_name) github_login = self._github_login_cache.get(author_name)
if github_login: if github_login:
return github_login return github_login
pr_info = self._fetch_github_pr(git_log.pr_number) if git_log.pr_number:
if pr_info: pr_info = self._fetch_github_pr(git_log.pr_number)
github_login = pr_info["user"]["login"] if pr_info:
else: github_login = pr_info["user"]["login"]
github_login = author_name else:
github_login = author_name
# set cache # set cache
self._github_login_cache[author_name] = github_login self._github_login_cache[author_name] = github_login
return github_login return github_login
def _get_changelog_version_head(self): def _get_changelog_version_head(self) -> str:
return f"### {self._version} ({self._logs[0].time})" return f"### {self._version} ({self._logs[0].time})"
def __repr__(self): def __repr__(self) -> str:
result = f"\n{self._get_changelog_version_head()}\n" result = f"\n{self._get_changelog_version_head()}\n"
for i, log in enumerate(self._logs): for i, log in enumerate(self._logs):
github_login = self._get_github_login(log) github_login = self._get_github_login(log)
@ -131,6 +138,19 @@ class GitChangeLog:
print(f"\r {i}/{len(self._logs)}", end="", flush=True) print(f"\r {i}/{len(self._logs)}", end="", flush=True)
return result return result
def __iter__(self) -> Iterator[Dict[str, Any]]:
for log in self._logs:
yield {
"pr_number": log.pr_number,
"pr_link": f"https://github.com/apache/incubator-superset/pull/"
f"{log.pr_number}",
"message": log.message,
"time": log.time,
"author": log.author,
"email": log.author_email,
"sha": log.sha,
}
class GitLogs: class GitLogs:
""" """
@ -151,35 +171,36 @@ class GitLogs:
def logs(self) -> List[GitLog]: def logs(self) -> List[GitLog]:
return self._logs return self._logs
def fetch(self): def fetch(self) -> None:
self._logs = list(map(self._parse_log, self._git_logs()))[::-1] self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
def diff(self, git_logs: "GitLogs") -> List[GitLog]: def diff(self, git_logs: "GitLogs") -> List[GitLog]:
return [log for log in git_logs.logs if log not in self._logs] return [log for log in git_logs.logs if log not in self._logs]
def __repr__(self): def __repr__(self) -> str:
return f"{self._git_ref}, Log count:{len(self._logs)}" return f"{self._git_ref}, Log count:{len(self._logs)}"
def _git_get_current_head(self) -> str: @staticmethod
def _git_get_current_head() -> str:
output = os.popen("git status | head -1").read() output = os.popen("git status | head -1").read()
match = re.match("(?:HEAD detached at|On branch) (.*)", output) match = re.match("(?:HEAD detached at|On branch) (.*)", output)
if not match: if not match:
return "" return ""
return match.group(1) return match.group(1)
def _git_checkout(self, git_ref: str): def _git_checkout(self, git_ref: str) -> None:
os.popen(f"git checkout {git_ref}").read() os.popen(f"git checkout {git_ref}").read()
current_head = self._git_get_current_head() current_head = self._git_get_current_head()
if current_head != git_ref: if current_head != git_ref:
print(f"Could not checkout {git_ref}") print(f"Could not checkout {git_ref}")
exit(1) sys.exit(1)
def _git_logs(self) -> List[str]: def _git_logs(self) -> List[str]:
# let's get current git ref so we can revert it back # let's get current git ref so we can revert it back
current_git_ref = self._git_get_current_head() current_git_ref = self._git_get_current_head()
self._git_checkout(self._git_ref) self._git_checkout(self._git_ref)
output = ( output = (
os.popen('git --no-pager log --pretty=format:"%h|%an|%ad|%s|"') os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')
.read() .read()
.split("\n") .split("\n")
) )
@ -187,18 +208,20 @@ class GitLogs:
self._git_checkout(current_git_ref) self._git_checkout(current_git_ref)
return output return output
def _parse_log(self, log_item: str) -> GitLog: @staticmethod
def _parse_log(log_item: str) -> GitLog:
pr_number = None pr_number = None
split_log_item = log_item.split("|") split_log_item = log_item.split("|")
# parse the PR number from the log message # parse the PR number from the log message
match = re.match(".*\(\#(\d*)\)", split_log_item[3]) match = re.match(r".*\(\#(\d*)\)", split_log_item[4])
if match: if match:
pr_number = int(match.group(1)) pr_number = int(match.group(1))
return GitLog( return GitLog(
sha=split_log_item[0], sha=split_log_item[0],
author=split_log_item[1], author=split_log_item[1],
time=split_log_item[2], author_email=split_log_item[2],
message=split_log_item[3], time=split_log_item[3],
message=split_log_item[4],
pr_number=pr_number, pr_number=pr_number,
) )
@ -217,13 +240,9 @@ def print_title(message: str) -> None:
@click.group() @click.group()
@click.pass_context @click.pass_context
@click.option( @click.option("--previous_version", help="The previous release version", required=True)
"--previous_version", help="The previous release version", @click.option("--current_version", help="The current release version", required=True)
) def cli(ctx, previous_version: str, current_version: str) -> None:
@click.option(
"--current_version", help="The current release version",
)
def cli(ctx, previous_version: str, current_version: str):
""" Welcome to change log generator """ """ Welcome to change log generator """
previous_logs = GitLogs(previous_version) previous_logs = GitLogs(previous_version)
current_logs = GitLogs(current_version) current_logs = GitLogs(current_version)
@ -235,7 +254,7 @@ def cli(ctx, previous_version: str, current_version: str):
@cli.command("compare") @cli.command("compare")
@click.pass_obj @click.pass_obj
def compare(base_parameters): def compare(base_parameters: BaseParameters) -> None:
""" Compares both versions (by PR) """ """ Compares both versions (by PR) """
previous_logs = base_parameters.previous_logs previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs current_logs = base_parameters.current_logs
@ -255,14 +274,33 @@ def compare(base_parameters):
@cli.command("changelog") @cli.command("changelog")
@click.option(
"--csv", help="The csv filename to export the changelog to",
)
@click.pass_obj @click.pass_obj
def changelog(base_parameters): def change_log(base_parameters: BaseParameters, csv: str) -> None:
""" Outputs a changelog (by PR) """ """ Outputs a changelog (by PR) """
previous_logs = base_parameters.previous_logs previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs current_logs = base_parameters.current_logs
previous_diff_logs = previous_logs.diff(current_logs) previous_diff_logs = previous_logs.diff(current_logs)
print("Fetching github usernames, this may take a while:") logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])
print(GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])) if csv:
with open(csv, "w") as csv_file:
log_items = list(logs)
field_names = log_items[0].keys()
writer = lib_csv.DictWriter(
csv_file,
delimiter=",",
quotechar='"',
quoting=lib_csv.QUOTE_ALL,
fieldnames=field_names,
)
writer.writeheader()
for log in logs:
writer.writerow(log)
else:
print("Fetching github usernames, this may take a while:")
print(logs)
cli() cli()