mirror of
https://github.com/apache/superset.git
synced 2024-09-12 16:49:40 -04:00
254 lines
7.6 KiB
Python
254 lines
7.6 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||
|
# or more contributor license agreements. See the NOTICE file
|
||
|
# distributed with this work for additional information
|
||
|
# regarding copyright ownership. The ASF licenses this file
|
||
|
# to you under the Apache License, Version 2.0 (the
|
||
|
# "License"); you may not use this file except in compliance
|
||
|
# with the License. You may obtain a copy of the License at
|
||
|
#
|
||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
#
|
||
|
# Unless required by applicable law or agreed to in writing,
|
||
|
# software distributed under the License is distributed on an
|
||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||
|
# KIND, either express or implied. See the License for the
|
||
|
# specific language governing permissions and limitations
|
||
|
# under the License.
|
||
|
import os
|
||
|
import re
|
||
|
import subprocess
|
||
|
from textwrap import dedent
|
||
|
|
||
|
import click
|
||
|
|
||
|
REPO = "apache/superset"
|
||
|
CACHE_REPO = f"{REPO}-cache"
|
||
|
BASE_PY_IMAGE = "3.9-slim-bookworm"
|
||
|
|
||
|
|
||
|
def run_cmd(command: str) -> str:
|
||
|
process = subprocess.Popen(
|
||
|
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
|
||
|
)
|
||
|
|
||
|
output = ""
|
||
|
if process.stdout is not None:
|
||
|
for line in iter(process.stdout.readline, ""):
|
||
|
print(line.strip()) # Print the line to stdout in real-time
|
||
|
output += line
|
||
|
|
||
|
process.wait() # Wait for the subprocess to finish
|
||
|
return output
|
||
|
|
||
|
|
||
|
def get_git_sha() -> str:
|
||
|
return run_cmd("git rev-parse HEAD").strip()
|
||
|
|
||
|
|
||
|
def get_build_context_ref(build_context: str) -> str:
|
||
|
"""
|
||
|
Given a context, return a ref:
|
||
|
- if context is pull_request, return the PR's id
|
||
|
- if context is push, return the branch
|
||
|
- if context is release, return the release ref
|
||
|
"""
|
||
|
|
||
|
event = os.getenv("GITHUB_EVENT_NAME")
|
||
|
github_ref = os.getenv("GITHUB_REF", "")
|
||
|
|
||
|
if event == "pull_request":
|
||
|
github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
|
||
|
return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
|
||
|
elif event == "release":
|
||
|
return re.sub("refs/tags/", "", github_ref)[:40]
|
||
|
elif event == "push":
|
||
|
return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
|
||
|
return ""
|
||
|
|
||
|
|
||
|
def is_latest_release(release: str) -> bool:
|
||
|
output = run_cmd(f"./scripts/tag_latest_release.sh {release} --dry-run") or ""
|
||
|
return "SKIP_TAG::false" in output
|
||
|
|
||
|
|
||
|
def make_docker_tag(l: list[str]) -> str:
|
||
|
return f"{REPO}:" + "-".join([o for o in l if o])
|
||
|
|
||
|
|
||
|
def get_docker_tags(
|
||
|
build_preset: str,
|
||
|
build_platform: str,
|
||
|
sha: str,
|
||
|
build_context: str,
|
||
|
build_context_ref: str,
|
||
|
) -> set[str]:
|
||
|
"""
|
||
|
Return a set of tags given a given build context
|
||
|
"""
|
||
|
tags: set[str] = set()
|
||
|
tag_chunks: list[str] = []
|
||
|
|
||
|
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
|
||
|
|
||
|
is_latest = is_latest_release(build_context_ref)
|
||
|
|
||
|
if build_preset != "lean":
|
||
|
# Always add the preset_build name if different from default (lean)
|
||
|
tag_chunks += [build_preset]
|
||
|
|
||
|
if short_build_platform != "amd":
|
||
|
# Always a platform indicator if different from default (amd)
|
||
|
tag_chunks += [short_build_platform]
|
||
|
|
||
|
# Always craft a tag for the SHA
|
||
|
tags.add(make_docker_tag([sha] + tag_chunks))
|
||
|
# also a short SHA, cause it's nice
|
||
|
tags.add(make_docker_tag([sha[:7]] + tag_chunks))
|
||
|
|
||
|
if build_context == "release":
|
||
|
# add a release tag
|
||
|
tags.add(make_docker_tag([build_context_ref] + tag_chunks))
|
||
|
if is_latest:
|
||
|
# add a latest tag
|
||
|
tags.add(make_docker_tag(["latest"] + tag_chunks))
|
||
|
elif build_context == "push" and build_context_ref == "master":
|
||
|
tags.add(make_docker_tag(["master"] + tag_chunks))
|
||
|
return tags
|
||
|
|
||
|
|
||
|
def get_docker_command(
|
||
|
build_preset: str,
|
||
|
build_platform: str,
|
||
|
is_authenticated: bool,
|
||
|
sha: str,
|
||
|
build_context: str,
|
||
|
build_context_ref: str,
|
||
|
) -> str:
|
||
|
tag = ""
|
||
|
build_target = ""
|
||
|
py_ver = BASE_PY_IMAGE
|
||
|
docker_context = "."
|
||
|
|
||
|
if build_preset == "dev":
|
||
|
build_target = "dev"
|
||
|
elif build_preset == "lean":
|
||
|
build_target = "lean"
|
||
|
elif build_preset == "py310":
|
||
|
build_target = "lean"
|
||
|
py_ver = "3.10-slim-bookworm"
|
||
|
elif build_preset == "websocket":
|
||
|
build_target = ""
|
||
|
docker_context = "superset-websocket"
|
||
|
elif build_preset == "ci":
|
||
|
build_target = "ci"
|
||
|
elif build_preset == "dockerize":
|
||
|
build_target = ""
|
||
|
docker_context = "-f dockerize.Dockerfile ."
|
||
|
else:
|
||
|
print(f"Invalid build preset: {build_preset}")
|
||
|
exit(1)
|
||
|
|
||
|
# Try to get context reference if missing
|
||
|
if not build_context_ref:
|
||
|
build_context_ref = get_build_context_ref(build_context)
|
||
|
|
||
|
tags = get_docker_tags(
|
||
|
build_preset,
|
||
|
build_platform,
|
||
|
sha,
|
||
|
build_context,
|
||
|
build_context_ref,
|
||
|
)
|
||
|
docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
|
||
|
|
||
|
docker_args = "--load" if not is_authenticated else "--push"
|
||
|
target_argument = f"--target {build_target}" if build_target else ""
|
||
|
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
|
||
|
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
|
||
|
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
|
||
|
cache_to_arg = (
|
||
|
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
|
||
|
)
|
||
|
build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
|
||
|
actor = os.getenv("GITHUB_ACTOR")
|
||
|
|
||
|
return dedent(
|
||
|
f"""\
|
||
|
docker buildx build \\
|
||
|
{docker_args} \\
|
||
|
{docker_tags} \\
|
||
|
{cache_from_arg} \\
|
||
|
{cache_to_arg} \\
|
||
|
{build_arg} \\
|
||
|
--platform {build_platform} \\
|
||
|
--label sha={sha} \\
|
||
|
--label target={build_target} \\
|
||
|
--label build_trigger={build_context} \\
|
||
|
--label base={py_ver} \\
|
||
|
--label build_actor={actor} \\
|
||
|
{docker_context}"""
|
||
|
)
|
||
|
|
||
|
|
||
|
@click.command()
|
||
|
@click.argument(
|
||
|
"build_preset",
|
||
|
type=click.Choice(["lean", "dev", "dockerize", "websocket", "py310", "ci"]),
|
||
|
)
|
||
|
@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
|
||
|
@click.option(
|
||
|
"--platform",
|
||
|
type=click.Choice(["linux/arm64", "linux/amd64"]),
|
||
|
default="linux/amd64",
|
||
|
)
|
||
|
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
|
||
|
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
|
||
|
def main(
|
||
|
build_preset: str,
|
||
|
build_context: str,
|
||
|
build_context_ref: str,
|
||
|
platform: str,
|
||
|
dry_run: bool,
|
||
|
) -> None:
|
||
|
"""
|
||
|
This script executes docker build and push commands based on given arguments.
|
||
|
"""
|
||
|
|
||
|
is_authenticated = (
|
||
|
True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
|
||
|
)
|
||
|
build_context_ref = get_build_context_ref(build_context)
|
||
|
|
||
|
docker_build_command = get_docker_command(
|
||
|
build_preset,
|
||
|
platform,
|
||
|
is_authenticated,
|
||
|
get_git_sha(),
|
||
|
build_context,
|
||
|
get_build_context_ref(build_context),
|
||
|
)
|
||
|
|
||
|
if not dry_run:
|
||
|
print("Executing Docker Build Command:")
|
||
|
print(docker_build_command)
|
||
|
script = ""
|
||
|
if os.getenv("DOCKERHUB_USER"):
|
||
|
script = dedent(
|
||
|
f"""\
|
||
|
docker logout
|
||
|
docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
|
||
|
DOCKER_ARGS="--push"
|
||
|
"""
|
||
|
)
|
||
|
script = script + docker_build_command
|
||
|
stdout = run_cmd(script)
|
||
|
else:
|
||
|
print("Dry Run - Docker Build Command:")
|
||
|
print(docker_build_command)
|
||
|
|
||
|
|
||
|
if __name__ == "__main__":
|
||
|
main()
|