pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(docker): allow for docker release builds to be multi-platform (#27055)

This commit is contained in:
Maxime Beauchemin 2024-02-08 20:58:36 -08:00 committed by GitHub
parent 5951f6ceb6
commit 13915bbb54
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 72 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/docker-release.yml vendored
View File

@ -2,7 +2,7 @@ name: Docker Publish Release
on:
release:
types: [published]
types: [published, edited]
# Can be triggered manually
workflow_dispatch:
@ -43,14 +43,6 @@ jobs:
strategy:
matrix:
build_preset: ["dev", "lean", "py310", "websocket", "dockerize"]
platform: ["linux/amd64", "linux/arm64"]
exclude:
# disabling because slow! no python wheels for arm/py39 and
# QEMU is slow!
- build_preset: "dev"
platform: "linux/arm64"
- build_preset: "lean"
platform: "linux/arm64"
fail-fast: false
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@ -88,8 +80,10 @@ jobs:
EVENT="release"
fi
pip install click
# Make a multi-platform image
./scripts/build_docker.py \
${{ matrix.build_preset }} \
"$EVENT" \
--build_context_ref "$RELEASE" \
--platform ${{ matrix.platform }} $FORCE_LATEST
--build_context_ref "$RELEASE" $FORCE_LATEST \
--platform "linux/arm64" \
--platform "linux/amd64"

7
.github/workflows/ephemeral-env.yml vendored
View File

@ -143,10 +143,9 @@ jobs:
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: superset-ci
SHA: ${{ steps.get-sha.outputs.sha }}
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$SHA
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up:
@ -181,7 +180,7 @@ jobs:
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=${{ steps.get-sha.outputs.sha }}
--image-ids imageTag=pr-${{ github.event.issue.number }}-ci
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
@ -204,7 +203,7 @@ jobs:
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
- name: Update env vars in the Amazon ECS task definition
run: |

42
scripts/build_docker.py
View File

@ -40,6 +40,10 @@ def run_cmd(command: str) -> str:
output += line
process.wait() # Wait for the subprocess to finish
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, command, output)
return output
@ -79,7 +83,7 @@ def make_docker_tag(l: list[str]) -> str:
def get_docker_tags(
build_preset: str,
build_platform: str,
build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
@ -91,17 +95,18 @@ def get_docker_tags(
tags: set[str] = set()
tag_chunks: list[str] = []
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
if short_build_platform != "amd":
# Always a platform indicator if different from default (amd)
tag_chunks += [short_build_platform]
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
if short_build_platform != "amd":
# Always a platform indicator if different from default (amd)
tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
@ -123,7 +128,7 @@ def get_docker_tags(
def get_docker_command(
build_preset: str,
build_platform: str,
build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
@ -160,7 +165,7 @@ def get_docker_command(
tags = get_docker_tags(
build_preset,
build_platform,
build_platforms,
sha,
build_context,
build_context_ref,
@ -170,8 +175,14 @@ def get_docker_command(
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
cache_ref = f"{CACHE_REPO}:{py_ver}"
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
platform_arg = "--platform " + ",".join(build_platforms)
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
@ -187,7 +198,7 @@ def get_docker_command(
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
--platform {build_platform} \\
{platform_arg} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
@ -206,10 +217,12 @@ def get_docker_command(
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
default="linux/amd64",
default=["linux/amd64"],
multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@click.option("--verbose", is_flag=True, help="Print more info")
@click.option(
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
)
@ -217,9 +230,10 @@ def main(
build_preset: str,
build_context: str,
build_context_ref: str,
platform: str,
platform: list[str],
dry_run: bool,
force_latest: bool,
verbose: bool,
) -> None:
"""
This script executes docker build and push commands based on given arguments.
@ -262,6 +276,8 @@ def main(
"""
)
script = script + docker_build_command
if verbose:
run_cmd("cat Dockerfile")
stdout = run_cmd(script)
else:
print("Dry Run - Docker Build Command:")

60
tests/unit_tests/scripts/docker_build.py
View File

@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool):
@pytest.mark.parametrize(
"build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
"build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
"linux/arm64",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"ci",
"linux/amd64",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
"linux/amd64",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/arm64",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/amd64",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool):
# old releases
(
"lean",
"linux/arm64",
["linux/arm64"],
SHA,
"release",
OLD_REL,
@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
"linux/amd64",
["linux/amd64"],
SHA,
"release",
OLD_REL,
@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/arm64",
["linux/arm64"],
SHA,
"release",
OLD_REL,
@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/amd64",
["linux/amd64"],
SHA,
"release",
OLD_REL,
@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool):
# new releases
(
"lean",
"linux/arm64",
["linux/arm64"],
SHA,
"release",
NEW_REL,
@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
"linux/amd64",
["linux/amd64"],
SHA,
"release",
NEW_REL,
@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/arm64",
["linux/arm64"],
SHA,
"release",
NEW_REL,
@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/amd64",
["linux/amd64"],
SHA,
"release",
NEW_REL,
@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool):
# merge on master
(
"lean",
"linux/arm64",
["linux/arm64"],
SHA,
"push",
"master",
@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
"linux/amd64",
["linux/amd64"],
SHA,
"push",
"master",
@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/arm64",
["linux/arm64"],
SHA,
"push",
"master",
@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
"linux/amd64",
["linux/amd64"],
SHA,
"push",
"master",
@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool):
],
)
def test_get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref
build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
"build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
"linux/amd64",
["linux/amd64"],
True,
SHA,
"push",
@ -251,18 +251,28 @@ def test_get_docker_tags(
),
(
"dev",
"linux/amd64",
["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
# multi-platform
(
"lean",
["linux/arm64", "linux/amd64"],
True,
SHA,
"push",
"master",
[f"--platform linux/arm64,linux/amd64"],
),
],
)
def test_get_docker_command(
build_preset,
build_platform,
build_platforms,
is_authenticated,
sha,
build_context,
@ -271,7 +281,7 @@ def test_get_docker_command(
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platform,
build_platforms,
is_authenticated,
sha,
build_context,