mirror of https://github.com/apache/superset.git
feat(releasing): support changelog csv export (#11893)
* feat(releasing): support changelog csv export * fix lint and type annotations
This commit is contained in:
parent
41738df77d
commit
f98ae014fa
|
@ -14,12 +14,16 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
# pylint: disable=no-value-for-parameter
|
||||||
|
|
||||||
|
import csv as lib_csv
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from time import sleep
|
from time import sleep
|
||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Dict, Iterator, List, Optional, Union
|
||||||
from urllib import request
|
from urllib import request
|
||||||
from urllib.error import HTTPError
|
from urllib.error import HTTPError
|
||||||
|
|
||||||
|
@ -37,6 +41,7 @@ class GitLog:
|
||||||
time: str
|
time: str
|
||||||
message: str
|
message: str
|
||||||
pr_number: Union[int, None] = None
|
pr_number: Union[int, None] = None
|
||||||
|
author_email: str = ""
|
||||||
|
|
||||||
def __eq__(self, other: object) -> bool:
|
def __eq__(self, other: object) -> bool:
|
||||||
""" A log entry is considered equal if it has the same PR number """
|
""" A log entry is considered equal if it has the same PR number """
|
||||||
|
@ -44,7 +49,7 @@ class GitLog:
|
||||||
return other.pr_number == self.pr_number
|
return other.pr_number == self.pr_number
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self) -> str:
|
||||||
return f"[{self.pr_number}]: {self.message} {self.time} {self.author}"
|
return f"[{self.pr_number}]: {self.message} {self.time} {self.author}"
|
||||||
|
|
||||||
|
|
||||||
|
@ -73,15 +78,16 @@ class GitChangeLog:
|
||||||
sleep(self._wait)
|
sleep(self._wait)
|
||||||
print()
|
print()
|
||||||
|
|
||||||
def _fetch_github_rate_limit(self) -> Dict[str, Any]:
|
@staticmethod
|
||||||
|
def _fetch_github_rate_limit() -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Fetches current github rate limit info
|
Fetches current github rate limit info
|
||||||
"""
|
"""
|
||||||
with request.urlopen(f"https://api.github.com/rate_limit") as response:
|
with request.urlopen("https://api.github.com/rate_limit") as response:
|
||||||
payload = json.loads(response.read())
|
payload = json.loads(response.read())
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
def _fetch_github_pr(self, pr_number) -> Dict[str, Any]:
|
def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Fetches a github PR info
|
Fetches a github PR info
|
||||||
"""
|
"""
|
||||||
|
@ -89,7 +95,7 @@ class GitChangeLog:
|
||||||
try:
|
try:
|
||||||
self._wait_github_rate_limit()
|
self._wait_github_rate_limit()
|
||||||
with request.urlopen(
|
with request.urlopen(
|
||||||
f"https://api.github.com/repos/apache/incubator-superset/pulls/"
|
"https://api.github.com/repos/apache/incubator-superset/pulls/"
|
||||||
f"{pr_number}"
|
f"{pr_number}"
|
||||||
) as response:
|
) as response:
|
||||||
payload = json.loads(response.read())
|
payload = json.loads(response.read())
|
||||||
|
@ -105,6 +111,7 @@ class GitChangeLog:
|
||||||
github_login = self._github_login_cache.get(author_name)
|
github_login = self._github_login_cache.get(author_name)
|
||||||
if github_login:
|
if github_login:
|
||||||
return github_login
|
return github_login
|
||||||
|
if git_log.pr_number:
|
||||||
pr_info = self._fetch_github_pr(git_log.pr_number)
|
pr_info = self._fetch_github_pr(git_log.pr_number)
|
||||||
if pr_info:
|
if pr_info:
|
||||||
github_login = pr_info["user"]["login"]
|
github_login = pr_info["user"]["login"]
|
||||||
|
@ -114,10 +121,10 @@ class GitChangeLog:
|
||||||
self._github_login_cache[author_name] = github_login
|
self._github_login_cache[author_name] = github_login
|
||||||
return github_login
|
return github_login
|
||||||
|
|
||||||
def _get_changelog_version_head(self):
|
def _get_changelog_version_head(self) -> str:
|
||||||
return f"### {self._version} ({self._logs[0].time})"
|
return f"### {self._version} ({self._logs[0].time})"
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self) -> str:
|
||||||
result = f"\n{self._get_changelog_version_head()}\n"
|
result = f"\n{self._get_changelog_version_head()}\n"
|
||||||
for i, log in enumerate(self._logs):
|
for i, log in enumerate(self._logs):
|
||||||
github_login = self._get_github_login(log)
|
github_login = self._get_github_login(log)
|
||||||
|
@ -131,6 +138,19 @@ class GitChangeLog:
|
||||||
print(f"\r {i}/{len(self._logs)}", end="", flush=True)
|
print(f"\r {i}/{len(self._logs)}", end="", flush=True)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def __iter__(self) -> Iterator[Dict[str, Any]]:
|
||||||
|
for log in self._logs:
|
||||||
|
yield {
|
||||||
|
"pr_number": log.pr_number,
|
||||||
|
"pr_link": f"https://github.com/apache/incubator-superset/pull/"
|
||||||
|
f"{log.pr_number}",
|
||||||
|
"message": log.message,
|
||||||
|
"time": log.time,
|
||||||
|
"author": log.author,
|
||||||
|
"email": log.author_email,
|
||||||
|
"sha": log.sha,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class GitLogs:
|
class GitLogs:
|
||||||
"""
|
"""
|
||||||
|
@ -151,35 +171,36 @@ class GitLogs:
|
||||||
def logs(self) -> List[GitLog]:
|
def logs(self) -> List[GitLog]:
|
||||||
return self._logs
|
return self._logs
|
||||||
|
|
||||||
def fetch(self):
|
def fetch(self) -> None:
|
||||||
self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
|
self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
|
||||||
|
|
||||||
def diff(self, git_logs: "GitLogs") -> List[GitLog]:
|
def diff(self, git_logs: "GitLogs") -> List[GitLog]:
|
||||||
return [log for log in git_logs.logs if log not in self._logs]
|
return [log for log in git_logs.logs if log not in self._logs]
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self) -> str:
|
||||||
return f"{self._git_ref}, Log count:{len(self._logs)}"
|
return f"{self._git_ref}, Log count:{len(self._logs)}"
|
||||||
|
|
||||||
def _git_get_current_head(self) -> str:
|
@staticmethod
|
||||||
|
def _git_get_current_head() -> str:
|
||||||
output = os.popen("git status | head -1").read()
|
output = os.popen("git status | head -1").read()
|
||||||
match = re.match("(?:HEAD detached at|On branch) (.*)", output)
|
match = re.match("(?:HEAD detached at|On branch) (.*)", output)
|
||||||
if not match:
|
if not match:
|
||||||
return ""
|
return ""
|
||||||
return match.group(1)
|
return match.group(1)
|
||||||
|
|
||||||
def _git_checkout(self, git_ref: str):
|
def _git_checkout(self, git_ref: str) -> None:
|
||||||
os.popen(f"git checkout {git_ref}").read()
|
os.popen(f"git checkout {git_ref}").read()
|
||||||
current_head = self._git_get_current_head()
|
current_head = self._git_get_current_head()
|
||||||
if current_head != git_ref:
|
if current_head != git_ref:
|
||||||
print(f"Could not checkout {git_ref}")
|
print(f"Could not checkout {git_ref}")
|
||||||
exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def _git_logs(self) -> List[str]:
|
def _git_logs(self) -> List[str]:
|
||||||
# let's get current git ref so we can revert it back
|
# let's get current git ref so we can revert it back
|
||||||
current_git_ref = self._git_get_current_head()
|
current_git_ref = self._git_get_current_head()
|
||||||
self._git_checkout(self._git_ref)
|
self._git_checkout(self._git_ref)
|
||||||
output = (
|
output = (
|
||||||
os.popen('git --no-pager log --pretty=format:"%h|%an|%ad|%s|"')
|
os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')
|
||||||
.read()
|
.read()
|
||||||
.split("\n")
|
.split("\n")
|
||||||
)
|
)
|
||||||
|
@ -187,18 +208,20 @@ class GitLogs:
|
||||||
self._git_checkout(current_git_ref)
|
self._git_checkout(current_git_ref)
|
||||||
return output
|
return output
|
||||||
|
|
||||||
def _parse_log(self, log_item: str) -> GitLog:
|
@staticmethod
|
||||||
|
def _parse_log(log_item: str) -> GitLog:
|
||||||
pr_number = None
|
pr_number = None
|
||||||
split_log_item = log_item.split("|")
|
split_log_item = log_item.split("|")
|
||||||
# parse the PR number from the log message
|
# parse the PR number from the log message
|
||||||
match = re.match(".*\(\#(\d*)\)", split_log_item[3])
|
match = re.match(r".*\(\#(\d*)\)", split_log_item[4])
|
||||||
if match:
|
if match:
|
||||||
pr_number = int(match.group(1))
|
pr_number = int(match.group(1))
|
||||||
return GitLog(
|
return GitLog(
|
||||||
sha=split_log_item[0],
|
sha=split_log_item[0],
|
||||||
author=split_log_item[1],
|
author=split_log_item[1],
|
||||||
time=split_log_item[2],
|
author_email=split_log_item[2],
|
||||||
message=split_log_item[3],
|
time=split_log_item[3],
|
||||||
|
message=split_log_item[4],
|
||||||
pr_number=pr_number,
|
pr_number=pr_number,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -217,13 +240,9 @@ def print_title(message: str) -> None:
|
||||||
|
|
||||||
@click.group()
|
@click.group()
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
@click.option(
|
@click.option("--previous_version", help="The previous release version", required=True)
|
||||||
"--previous_version", help="The previous release version",
|
@click.option("--current_version", help="The current release version", required=True)
|
||||||
)
|
def cli(ctx, previous_version: str, current_version: str) -> None:
|
||||||
@click.option(
|
|
||||||
"--current_version", help="The current release version",
|
|
||||||
)
|
|
||||||
def cli(ctx, previous_version: str, current_version: str):
|
|
||||||
""" Welcome to change log generator """
|
""" Welcome to change log generator """
|
||||||
previous_logs = GitLogs(previous_version)
|
previous_logs = GitLogs(previous_version)
|
||||||
current_logs = GitLogs(current_version)
|
current_logs = GitLogs(current_version)
|
||||||
|
@ -235,7 +254,7 @@ def cli(ctx, previous_version: str, current_version: str):
|
||||||
|
|
||||||
@cli.command("compare")
|
@cli.command("compare")
|
||||||
@click.pass_obj
|
@click.pass_obj
|
||||||
def compare(base_parameters):
|
def compare(base_parameters: BaseParameters) -> None:
|
||||||
""" Compares both versions (by PR) """
|
""" Compares both versions (by PR) """
|
||||||
previous_logs = base_parameters.previous_logs
|
previous_logs = base_parameters.previous_logs
|
||||||
current_logs = base_parameters.current_logs
|
current_logs = base_parameters.current_logs
|
||||||
|
@ -255,14 +274,33 @@ def compare(base_parameters):
|
||||||
|
|
||||||
|
|
||||||
@cli.command("changelog")
|
@cli.command("changelog")
|
||||||
|
@click.option(
|
||||||
|
"--csv", help="The csv filename to export the changelog to",
|
||||||
|
)
|
||||||
@click.pass_obj
|
@click.pass_obj
|
||||||
def changelog(base_parameters):
|
def change_log(base_parameters: BaseParameters, csv: str) -> None:
|
||||||
""" Outputs a changelog (by PR) """
|
""" Outputs a changelog (by PR) """
|
||||||
previous_logs = base_parameters.previous_logs
|
previous_logs = base_parameters.previous_logs
|
||||||
current_logs = base_parameters.current_logs
|
current_logs = base_parameters.current_logs
|
||||||
previous_diff_logs = previous_logs.diff(current_logs)
|
previous_diff_logs = previous_logs.diff(current_logs)
|
||||||
|
logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])
|
||||||
|
if csv:
|
||||||
|
with open(csv, "w") as csv_file:
|
||||||
|
log_items = list(logs)
|
||||||
|
field_names = log_items[0].keys()
|
||||||
|
writer = lib_csv.DictWriter(
|
||||||
|
csv_file,
|
||||||
|
delimiter=",",
|
||||||
|
quotechar='"',
|
||||||
|
quoting=lib_csv.QUOTE_ALL,
|
||||||
|
fieldnames=field_names,
|
||||||
|
)
|
||||||
|
writer.writeheader()
|
||||||
|
for log in logs:
|
||||||
|
writer.writerow(log)
|
||||||
|
else:
|
||||||
print("Fetching github usernames, this may take a while:")
|
print("Fetching github usernames, this may take a while:")
|
||||||
print(GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1]))
|
print(logs)
|
||||||
|
|
||||||
|
|
||||||
cli()
|
cli()
|
||||||
|
|
Loading…
Reference in New Issue