#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import os import re import subprocess from textwrap import dedent import click REPO = "apache/superset" CACHE_REPO = f"{REPO}-cache" BASE_PY_IMAGE = "3.10-slim-bookworm" def run_cmd(command: str, raise_on_failure: bool = True) -> str: process = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) output = "" if process.stdout is not None: for line in iter(process.stdout.readline, ""): print(line.strip()) # Print the line to stdout in real-time output += line process.wait() # Wait for the subprocess to finish if process.returncode != 0 and raise_on_failure: raise subprocess.CalledProcessError(process.returncode, command, output) return output def get_git_sha() -> str: return run_cmd("git rev-parse HEAD").strip() def get_build_context_ref(build_context: str) -> str: """ Given a context, return a ref: - if context is pull_request, return the PR's id - if context is push, return the branch - if context is release, return the release ref """ event = os.getenv("GITHUB_EVENT_NAME") github_ref = os.getenv("GITHUB_REF", "") if event == "pull_request": github_head_ref = os.getenv("GITHUB_HEAD_REF", "") return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40] elif event == "release": return re.sub("refs/tags/", "", github_ref)[:40] elif event == "push": return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40] return "" def is_latest_release(release: str) -> bool: output = ( run_cmd( f"./scripts/tag_latest_release.sh {release} --dry-run", raise_on_failure=False, ) or "" ) return "SKIP_TAG::false" in output def make_docker_tag(l: list[str]) -> str: return f"{REPO}:" + "-".join([o for o in l if o]) def get_docker_tags( build_preset: str, build_platforms: list[str], sha: str, build_context: str, build_context_ref: str, force_latest: bool = False, ) -> set[str]: """ Return a set of tags given a given build context """ tags: set[str] = set() tag_chunks: list[str] = [] is_latest = is_latest_release(build_context_ref) if build_preset != "lean": # Always add the preset_build name if different from default (lean) tag_chunks += [build_preset] if len(build_platforms) == 1: build_platform = build_platforms[0] short_build_platform = build_platform.replace("linux/", "").replace("64", "") if short_build_platform != "amd": # Always a platform indicator if different from default (amd) tag_chunks += [short_build_platform] # Always craft a tag for the SHA tags.add(make_docker_tag([sha] + tag_chunks)) # also a short SHA, cause it's nice tags.add(make_docker_tag([sha[:7]] + tag_chunks)) if build_context == "release": # add a release tag tags.add(make_docker_tag([build_context_ref] + tag_chunks)) if is_latest or force_latest: # add a latest tag tags.add(make_docker_tag(["latest"] + tag_chunks)) elif build_context == "push" and build_context_ref == "master": tags.add(make_docker_tag(["master"] + tag_chunks)) elif build_context == "pull_request": tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks)) return tags def get_docker_command( build_preset: str, build_platforms: list[str], is_authenticated: bool, sha: str, build_context: str, build_context_ref: str, force_latest: bool = False, ) -> str: tag = "" build_target = "" py_ver = BASE_PY_IMAGE docker_context = "." if build_preset == "dev": build_target = "dev" elif build_preset == "lean": build_target = "lean" elif build_preset == "py311": build_target = "lean" py_ver = "3.11-slim-bookworm" elif build_preset == "websocket": build_target = "" docker_context = "superset-websocket" elif build_preset == "ci": build_target = "ci" elif build_preset == "dockerize": build_target = "" docker_context = "-f dockerize.Dockerfile ." else: print(f"Invalid build preset: {build_preset}") exit(1) # Try to get context reference if missing if not build_context_ref: build_context_ref = get_build_context_ref(build_context) tags = get_docker_tags( build_preset, build_platforms, sha, build_context, build_context_ref, force_latest, ) docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags]) docker_args = "--load" if not is_authenticated else "--push" target_argument = f"--target {build_target}" if build_target else "" cache_ref = f"{CACHE_REPO}:{py_ver}" if len(build_platforms) == 1: build_platform = build_platforms[0] short_build_platform = build_platform.replace("linux/", "").replace("64", "") cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" platform_arg = "--platform " + ",".join(build_platforms) cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}" cache_to_arg = ( f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else "" ) build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else "" actor = os.getenv("GITHUB_ACTOR") return dedent( f"""\ docker buildx build \\ {docker_args} \\ {docker_tags} \\ {cache_from_arg} \\ {cache_to_arg} \\ {build_arg} \\ {platform_arg} \\ {target_argument} \\ --label sha={sha} \\ --label target={build_target} \\ --label build_trigger={build_context} \\ --label base={py_ver} \\ --label build_actor={actor} \\ {docker_context}""" ) @click.command() @click.argument( "build_preset", type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]), ) @click.argument("build_context", type=click.Choice(["push", "pull_request", "release"])) @click.option( "--platform", type=click.Choice(["linux/arm64", "linux/amd64"]), default=["linux/amd64"], multiple=True, ) @click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") @click.option("--verbose", is_flag=True, help="Print more info") @click.option( "--force-latest", is_flag=True, help="Force the 'latest' tag on the release" ) def main( build_preset: str, build_context: str, build_context_ref: str, platform: list[str], dry_run: bool, force_latest: bool, verbose: bool, ) -> None: """ This script executes docker build and push commands based on given arguments. """ is_authenticated = ( True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False ) if force_latest and build_context != "release": print( "--force-latest can only be applied if the build context is set to 'release'" ) exit(1) if build_context == "release" and not build_context_ref.strip(): print("Release number has to be provided") exit(1) docker_build_command = get_docker_command( build_preset, platform, is_authenticated, get_git_sha(), build_context, build_context_ref, force_latest, ) if not dry_run: print("Executing Docker Build Command:") print(docker_build_command) script = "" if os.getenv("DOCKERHUB_USER"): script = dedent( f"""\ docker logout docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}" DOCKER_ARGS="--push" """ ) script = script + docker_build_command if verbose: run_cmd("cat Dockerfile") stdout = run_cmd(script) else: print("Dry Run - Docker Build Command:") print(docker_build_command) if __name__ == "__main__": main()