2024-01-31 02:03:25 -05:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
# distributed with this work for additional information
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
# under the License.
|
|
|
|
import os
|
|
|
|
import re
|
|
|
|
import subprocess
|
|
|
|
from textwrap import dedent
|
|
|
|
|
|
|
|
import click
|
|
|
|
|
|
|
|
REPO = "apache/superset"
|
|
|
|
CACHE_REPO = f"{REPO}-cache"
|
2024-04-02 12:47:58 -04:00
|
|
|
BASE_PY_IMAGE = "3.10-slim-bookworm"
|
2024-01-31 02:03:25 -05:00
|
|
|
|
|
|
|
|
2024-02-12 12:53:36 -05:00
|
|
|
def run_cmd(command: str, raise_on_failure: bool = True) -> str:
|
2024-01-31 02:03:25 -05:00
|
|
|
process = subprocess.Popen(
|
|
|
|
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
|
|
|
|
)
|
|
|
|
|
|
|
|
output = ""
|
|
|
|
if process.stdout is not None:
|
|
|
|
for line in iter(process.stdout.readline, ""):
|
|
|
|
print(line.strip()) # Print the line to stdout in real-time
|
|
|
|
output += line
|
|
|
|
|
|
|
|
process.wait() # Wait for the subprocess to finish
|
2024-02-08 23:58:36 -05:00
|
|
|
|
2024-02-12 12:53:36 -05:00
|
|
|
if process.returncode != 0 and raise_on_failure:
|
2024-02-08 23:58:36 -05:00
|
|
|
raise subprocess.CalledProcessError(process.returncode, command, output)
|
2024-01-31 02:03:25 -05:00
|
|
|
return output
|
|
|
|
|
|
|
|
|
|
|
|
def get_git_sha() -> str:
|
|
|
|
return run_cmd("git rev-parse HEAD").strip()
|
|
|
|
|
|
|
|
|
|
|
|
def get_build_context_ref(build_context: str) -> str:
|
|
|
|
"""
|
|
|
|
Given a context, return a ref:
|
|
|
|
- if context is pull_request, return the PR's id
|
|
|
|
- if context is push, return the branch
|
|
|
|
- if context is release, return the release ref
|
|
|
|
"""
|
|
|
|
|
|
|
|
event = os.getenv("GITHUB_EVENT_NAME")
|
|
|
|
github_ref = os.getenv("GITHUB_REF", "")
|
|
|
|
|
|
|
|
if event == "pull_request":
|
|
|
|
github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
|
|
|
|
return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
|
|
|
|
elif event == "release":
|
|
|
|
return re.sub("refs/tags/", "", github_ref)[:40]
|
|
|
|
elif event == "push":
|
|
|
|
return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
|
|
def is_latest_release(release: str) -> bool:
|
2024-02-12 12:53:36 -05:00
|
|
|
output = (
|
|
|
|
run_cmd(
|
|
|
|
f"./scripts/tag_latest_release.sh {release} --dry-run",
|
|
|
|
raise_on_failure=False,
|
|
|
|
)
|
|
|
|
or ""
|
|
|
|
)
|
2024-01-31 02:03:25 -05:00
|
|
|
return "SKIP_TAG::false" in output
|
|
|
|
|
|
|
|
|
2024-04-24 20:19:53 -04:00
|
|
|
def make_docker_tag(l: list[str]) -> str: # noqa: E741
|
2024-01-31 02:03:25 -05:00
|
|
|
return f"{REPO}:" + "-".join([o for o in l if o])
|
|
|
|
|
|
|
|
|
|
|
|
def get_docker_tags(
|
|
|
|
build_preset: str,
|
2024-02-08 23:58:36 -05:00
|
|
|
build_platforms: list[str],
|
2024-01-31 02:03:25 -05:00
|
|
|
sha: str,
|
|
|
|
build_context: str,
|
|
|
|
build_context_ref: str,
|
2024-02-01 21:04:23 -05:00
|
|
|
force_latest: bool = False,
|
2024-01-31 02:03:25 -05:00
|
|
|
) -> set[str]:
|
|
|
|
"""
|
|
|
|
Return a set of tags given a given build context
|
|
|
|
"""
|
|
|
|
tags: set[str] = set()
|
|
|
|
tag_chunks: list[str] = []
|
|
|
|
|
|
|
|
is_latest = is_latest_release(build_context_ref)
|
|
|
|
|
|
|
|
if build_preset != "lean":
|
|
|
|
# Always add the preset_build name if different from default (lean)
|
|
|
|
tag_chunks += [build_preset]
|
|
|
|
|
2024-02-08 23:58:36 -05:00
|
|
|
if len(build_platforms) == 1:
|
|
|
|
build_platform = build_platforms[0]
|
|
|
|
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
|
|
|
|
if short_build_platform != "amd":
|
|
|
|
# Always a platform indicator if different from default (amd)
|
|
|
|
tag_chunks += [short_build_platform]
|
2024-01-31 02:03:25 -05:00
|
|
|
|
|
|
|
# Always craft a tag for the SHA
|
|
|
|
tags.add(make_docker_tag([sha] + tag_chunks))
|
|
|
|
# also a short SHA, cause it's nice
|
|
|
|
tags.add(make_docker_tag([sha[:7]] + tag_chunks))
|
|
|
|
|
|
|
|
if build_context == "release":
|
|
|
|
# add a release tag
|
|
|
|
tags.add(make_docker_tag([build_context_ref] + tag_chunks))
|
2024-02-01 21:04:23 -05:00
|
|
|
if is_latest or force_latest:
|
2024-01-31 02:03:25 -05:00
|
|
|
# add a latest tag
|
|
|
|
tags.add(make_docker_tag(["latest"] + tag_chunks))
|
|
|
|
elif build_context == "push" and build_context_ref == "master":
|
|
|
|
tags.add(make_docker_tag(["master"] + tag_chunks))
|
2024-02-01 21:04:23 -05:00
|
|
|
elif build_context == "pull_request":
|
|
|
|
tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
|
2024-01-31 02:03:25 -05:00
|
|
|
return tags
|
|
|
|
|
|
|
|
|
|
|
|
def get_docker_command(
|
|
|
|
build_preset: str,
|
2024-02-08 23:58:36 -05:00
|
|
|
build_platforms: list[str],
|
2024-01-31 02:03:25 -05:00
|
|
|
is_authenticated: bool,
|
|
|
|
sha: str,
|
|
|
|
build_context: str,
|
|
|
|
build_context_ref: str,
|
2024-02-01 21:04:23 -05:00
|
|
|
force_latest: bool = False,
|
2024-01-31 02:03:25 -05:00
|
|
|
) -> str:
|
2024-04-24 20:19:53 -04:00
|
|
|
tag = "" # noqa: F841
|
2024-01-31 02:03:25 -05:00
|
|
|
build_target = ""
|
|
|
|
py_ver = BASE_PY_IMAGE
|
|
|
|
docker_context = "."
|
|
|
|
|
|
|
|
if build_preset == "dev":
|
|
|
|
build_target = "dev"
|
|
|
|
elif build_preset == "lean":
|
|
|
|
build_target = "lean"
|
2024-04-02 12:47:58 -04:00
|
|
|
elif build_preset == "py311":
|
2024-01-31 02:03:25 -05:00
|
|
|
build_target = "lean"
|
2024-04-02 12:47:58 -04:00
|
|
|
py_ver = "3.11-slim-bookworm"
|
2024-01-31 02:03:25 -05:00
|
|
|
elif build_preset == "websocket":
|
|
|
|
build_target = ""
|
|
|
|
docker_context = "superset-websocket"
|
|
|
|
elif build_preset == "ci":
|
|
|
|
build_target = "ci"
|
|
|
|
elif build_preset == "dockerize":
|
|
|
|
build_target = ""
|
|
|
|
docker_context = "-f dockerize.Dockerfile ."
|
|
|
|
else:
|
|
|
|
print(f"Invalid build preset: {build_preset}")
|
|
|
|
exit(1)
|
|
|
|
|
|
|
|
# Try to get context reference if missing
|
|
|
|
if not build_context_ref:
|
|
|
|
build_context_ref = get_build_context_ref(build_context)
|
|
|
|
|
|
|
|
tags = get_docker_tags(
|
|
|
|
build_preset,
|
2024-02-08 23:58:36 -05:00
|
|
|
build_platforms,
|
2024-01-31 02:03:25 -05:00
|
|
|
sha,
|
|
|
|
build_context,
|
|
|
|
build_context_ref,
|
2024-02-01 21:04:23 -05:00
|
|
|
force_latest,
|
2024-01-31 02:03:25 -05:00
|
|
|
)
|
|
|
|
docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
|
|
|
|
|
|
|
|
docker_args = "--load" if not is_authenticated else "--push"
|
|
|
|
target_argument = f"--target {build_target}" if build_target else ""
|
2024-02-08 23:58:36 -05:00
|
|
|
|
|
|
|
cache_ref = f"{CACHE_REPO}:{py_ver}"
|
|
|
|
if len(build_platforms) == 1:
|
|
|
|
build_platform = build_platforms[0]
|
|
|
|
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
|
|
|
|
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
|
|
|
|
platform_arg = "--platform " + ",".join(build_platforms)
|
|
|
|
|
2024-01-31 02:03:25 -05:00
|
|
|
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
|
|
|
|
cache_to_arg = (
|
|
|
|
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
|
|
|
|
)
|
|
|
|
build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
|
|
|
|
actor = os.getenv("GITHUB_ACTOR")
|
|
|
|
|
|
|
|
return dedent(
|
|
|
|
f"""\
|
|
|
|
docker buildx build \\
|
|
|
|
{docker_args} \\
|
|
|
|
{docker_tags} \\
|
|
|
|
{cache_from_arg} \\
|
|
|
|
{cache_to_arg} \\
|
|
|
|
{build_arg} \\
|
2024-02-08 23:58:36 -05:00
|
|
|
{platform_arg} \\
|
2024-02-14 19:58:47 -05:00
|
|
|
{target_argument} \\
|
2024-01-31 02:03:25 -05:00
|
|
|
--label sha={sha} \\
|
|
|
|
--label target={build_target} \\
|
|
|
|
--label build_trigger={build_context} \\
|
|
|
|
--label base={py_ver} \\
|
|
|
|
--label build_actor={actor} \\
|
|
|
|
{docker_context}"""
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
@click.command()
|
|
|
|
@click.argument(
|
|
|
|
"build_preset",
|
2024-04-02 12:47:58 -04:00
|
|
|
type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]),
|
2024-01-31 02:03:25 -05:00
|
|
|
)
|
|
|
|
@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
|
|
|
|
@click.option(
|
|
|
|
"--platform",
|
|
|
|
type=click.Choice(["linux/arm64", "linux/amd64"]),
|
2024-02-08 23:58:36 -05:00
|
|
|
default=["linux/amd64"],
|
|
|
|
multiple=True,
|
2024-01-31 02:03:25 -05:00
|
|
|
)
|
|
|
|
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
|
|
|
|
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
|
2024-02-08 23:58:36 -05:00
|
|
|
@click.option("--verbose", is_flag=True, help="Print more info")
|
2024-02-01 21:04:23 -05:00
|
|
|
@click.option(
|
|
|
|
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
|
|
|
|
)
|
2024-01-31 02:03:25 -05:00
|
|
|
def main(
|
|
|
|
build_preset: str,
|
|
|
|
build_context: str,
|
|
|
|
build_context_ref: str,
|
2024-02-08 23:58:36 -05:00
|
|
|
platform: list[str],
|
2024-01-31 02:03:25 -05:00
|
|
|
dry_run: bool,
|
2024-02-01 21:04:23 -05:00
|
|
|
force_latest: bool,
|
2024-02-08 23:58:36 -05:00
|
|
|
verbose: bool,
|
2024-01-31 02:03:25 -05:00
|
|
|
) -> None:
|
|
|
|
"""
|
|
|
|
This script executes docker build and push commands based on given arguments.
|
|
|
|
"""
|
|
|
|
|
|
|
|
is_authenticated = (
|
|
|
|
True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
|
|
|
|
)
|
2024-02-01 21:04:23 -05:00
|
|
|
|
|
|
|
if force_latest and build_context != "release":
|
|
|
|
print(
|
|
|
|
"--force-latest can only be applied if the build context is set to 'release'"
|
|
|
|
)
|
|
|
|
exit(1)
|
|
|
|
|
|
|
|
if build_context == "release" and not build_context_ref.strip():
|
|
|
|
print("Release number has to be provided")
|
|
|
|
exit(1)
|
2024-01-31 02:03:25 -05:00
|
|
|
|
|
|
|
docker_build_command = get_docker_command(
|
|
|
|
build_preset,
|
|
|
|
platform,
|
|
|
|
is_authenticated,
|
|
|
|
get_git_sha(),
|
|
|
|
build_context,
|
2024-02-01 21:04:23 -05:00
|
|
|
build_context_ref,
|
|
|
|
force_latest,
|
2024-01-31 02:03:25 -05:00
|
|
|
)
|
|
|
|
|
|
|
|
if not dry_run:
|
|
|
|
print("Executing Docker Build Command:")
|
|
|
|
print(docker_build_command)
|
|
|
|
script = ""
|
|
|
|
if os.getenv("DOCKERHUB_USER"):
|
|
|
|
script = dedent(
|
|
|
|
f"""\
|
|
|
|
docker logout
|
|
|
|
docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
|
|
|
|
DOCKER_ARGS="--push"
|
|
|
|
"""
|
|
|
|
)
|
|
|
|
script = script + docker_build_command
|
2024-02-08 23:58:36 -05:00
|
|
|
if verbose:
|
|
|
|
run_cmd("cat Dockerfile")
|
2024-04-24 20:19:53 -04:00
|
|
|
stdout = run_cmd(script) # noqa: F841
|
2024-01-31 02:03:25 -05:00
|
|
|
else:
|
|
|
|
print("Dry Run - Docker Build Command:")
|
|
|
|
print(docker_build_command)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
main()
|