pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(docker): allow for docker release builds to be multi-platform (#27055)

This commit is contained in:
Maxime Beauchemin 2024-02-08 20:58:36 -08:00 committed by GitHub
parent 5951f6ceb6
commit 13915bbb54
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 72 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/docker-release.yml vendored
View File

@ -2,7 +2,7 @@ name: Docker Publish Release
on: on:
release: release:
types: [published] types: [published, edited]
# Can be triggered manually # Can be triggered manually
workflow_dispatch: workflow_dispatch:
@ -43,14 +43,6 @@ jobs:
strategy: strategy:
matrix: matrix:
build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] build_preset: ["dev", "lean", "py310", "websocket", "dockerize"]
platform: ["linux/amd64", "linux/arm64"]
exclude:
# disabling because slow! no python wheels for arm/py39 and
# QEMU is slow!
- build_preset: "dev"
platform: "linux/arm64"
- build_preset: "lean"
platform: "linux/arm64"
fail-fast: false fail-fast: false
steps: steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@ -88,8 +80,10 @@ jobs:
EVENT="release" EVENT="release"
fi fi
pip install click pip install click
# Make a multi-platform image
./scripts/build_docker.py \ ./scripts/build_docker.py \
${{ matrix.build_preset }} \ ${{ matrix.build_preset }} \
"$EVENT" \ "$EVENT" \
--build_context_ref "$RELEASE" \ --build_context_ref "$RELEASE" $FORCE_LATEST \
--platform ${{ matrix.platform }} $FORCE_LATEST --platform "linux/arm64" \
--platform "linux/amd64"

7
.github/workflows/ephemeral-env.yml vendored
View File

@ -143,10 +143,9 @@ jobs:
env: env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: superset-ci ECR_REPOSITORY: superset-ci
SHA: ${{ steps.get-sha.outputs.sha }}
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
run: | run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$SHA docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up: ephemeral-env-up:
@ -181,7 +180,7 @@ jobs:
aws ecr describe-images \ aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \ --repository-name superset-ci \
--image-ids imageTag=${{ steps.get-sha.outputs.sha }} --image-ids imageTag=pr-${{ github.event.issue.number }}-ci
- name: Fail on missing container image - name: Fail on missing container image
if: steps.check-image.outcome == 'failure' if: steps.check-image.outcome == 'failure'
@ -204,7 +203,7 @@ jobs:
with: with:
task-definition: .github/workflows/ecs-task-definition.json task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }} image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
- name: Update env vars in the Amazon ECS task definition - name: Update env vars in the Amazon ECS task definition
run: | run: |

42
scripts/build_docker.py
View File

@ -40,6 +40,10 @@ def run_cmd(command: str) -> str:
output += line output += line
process.wait() # Wait for the subprocess to finish process.wait() # Wait for the subprocess to finish
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, command, output)
return output return output
@ -79,7 +83,7 @@ def make_docker_tag(l: list[str]) -> str:
def get_docker_tags( def get_docker_tags(
build_preset: str, build_preset: str,
build_platform: str, build_platforms: list[str],
sha: str, sha: str,
build_context: str, build_context: str,
build_context_ref: str, build_context_ref: str,
@ -91,17 +95,18 @@ def get_docker_tags(
tags: set[str] = set() tags: set[str] = set()
tag_chunks: list[str] = [] tag_chunks: list[str] = []
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
is_latest = is_latest_release(build_context_ref) is_latest = is_latest_release(build_context_ref)
if build_preset != "lean": if build_preset != "lean":
# Always add the preset_build name if different from default (lean) # Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset] tag_chunks += [build_preset]
if short_build_platform != "amd": if len(build_platforms) == 1:
# Always a platform indicator if different from default (amd) build_platform = build_platforms[0]
tag_chunks += [short_build_platform] short_build_platform = build_platform.replace("linux/", "").replace("64", "")
if short_build_platform != "amd":
# Always a platform indicator if different from default (amd)
tag_chunks += [short_build_platform]
# Always craft a tag for the SHA # Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks)) tags.add(make_docker_tag([sha] + tag_chunks))
@ -123,7 +128,7 @@ def get_docker_tags(
def get_docker_command( def get_docker_command(
build_preset: str, build_preset: str,
build_platform: str, build_platforms: list[str],
is_authenticated: bool, is_authenticated: bool,
sha: str, sha: str,
build_context: str, build_context: str,
@ -160,7 +165,7 @@ def get_docker_command(
tags = get_docker_tags( tags = get_docker_tags(
build_preset, build_preset,
build_platform, build_platforms,
sha, sha,
build_context, build_context,
build_context_ref, build_context_ref,
@ -170,8 +175,14 @@ def get_docker_command(
docker_args = "--load" if not is_authenticated else "--push" docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else "" target_argument = f"--target {build_target}" if build_target else ""
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" cache_ref = f"{CACHE_REPO}:{py_ver}"
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
platform_arg = "--platform " + ",".join(build_platforms)
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}" cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = ( cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else "" f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
@ -187,7 +198,7 @@ def get_docker_command(
{cache_from_arg} \\ {cache_from_arg} \\
{cache_to_arg} \\ {cache_to_arg} \\
{build_arg} \\ {build_arg} \\
--platform {build_platform} \\ {platform_arg} \\
--label sha={sha} \\ --label sha={sha} \\
--label target={build_target} \\ --label target={build_target} \\
--label build_trigger={build_context} \\ --label build_trigger={build_context} \\
@ -206,10 +217,12 @@ def get_docker_command(
@click.option( @click.option(
"--platform", "--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]), type=click.Choice(["linux/arm64", "linux/amd64"]),
default="linux/amd64", default=["linux/amd64"],
multiple=True,
) )
@click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@click.option("--verbose", is_flag=True, help="Print more info")
@click.option( @click.option(
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release" "--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
) )
@ -217,9 +230,10 @@ def main(
build_preset: str, build_preset: str,
build_context: str, build_context: str,
build_context_ref: str, build_context_ref: str,
platform: str, platform: list[str],
dry_run: bool, dry_run: bool,
force_latest: bool, force_latest: bool,
verbose: bool,
) -> None: ) -> None:
""" """
This script executes docker build and push commands based on given arguments. This script executes docker build and push commands based on given arguments.
@ -262,6 +276,8 @@ def main(
""" """
) )
script = script + docker_build_command script = script + docker_build_command
if verbose:
run_cmd("cat Dockerfile")
stdout = run_cmd(script) stdout = run_cmd(script)
else: else:
print("Dry Run - Docker Build Command:") print("Dry Run - Docker Build Command:")

60
tests/unit_tests/scripts/docker_build.py
View File

@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"build_preset, build_platform, sha, build_context, build_context_ref, expected_tags", "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[ [
# PRs # PRs
( (
"lean", "lean",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"pull_request", "pull_request",
PR_ID, PR_ID,
@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"ci", "ci",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"pull_request", "pull_request",
PR_ID, PR_ID,
@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"lean", "lean",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"pull_request", "pull_request",
PR_ID, PR_ID,
@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"pull_request", "pull_request",
PR_ID, PR_ID,
@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"pull_request", "pull_request",
PR_ID, PR_ID,
@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool):
# old releases # old releases
( (
"lean", "lean",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"release", "release",
OLD_REL, OLD_REL,
@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"lean", "lean",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"release", "release",
OLD_REL, OLD_REL,
@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"release", "release",
OLD_REL, OLD_REL,
@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"release", "release",
OLD_REL, OLD_REL,
@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool):
# new releases # new releases
( (
"lean", "lean",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"release", "release",
NEW_REL, NEW_REL,
@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"lean", "lean",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"release", "release",
NEW_REL, NEW_REL,
@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"release", "release",
NEW_REL, NEW_REL,
@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"release", "release",
NEW_REL, NEW_REL,
@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool):
# merge on master # merge on master
( (
"lean", "lean",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"push", "push",
"master", "master",
@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"lean", "lean",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"push", "push",
"master", "master",
@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/arm64", ["linux/arm64"],
SHA, SHA,
"push", "push",
"master", "master",
@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool):
), ),
( (
"dev", "dev",
"linux/amd64", ["linux/amd64"],
SHA, SHA,
"push", "push",
"master", "master",
@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool):
], ],
) )
def test_get_docker_tags( def test_get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref, expected_tags build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
): ):
tags = docker_utils.get_docker_tags( tags = docker_utils.get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref build_preset, build_platforms, sha, build_context, build_context_ref
) )
for tag in expected_tags: for tag in expected_tags:
assert tag in tags assert tag in tags
@pytest.mark.parametrize( @pytest.mark.parametrize(
"build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains", "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[ [
( (
"lean", "lean",
"linux/amd64", ["linux/amd64"],
True, True,
SHA, SHA,
"push", "push",
@ -251,18 +251,28 @@ def test_get_docker_tags(
), ),
( (
"dev", "dev",
"linux/amd64", ["linux/amd64"],
False, False,
SHA, SHA,
"push", "push",
"master", "master",
["--load", f"-t {REPO}:master-dev "], ["--load", f"-t {REPO}:master-dev "],
), ),
# multi-platform
(
"lean",
["linux/arm64", "linux/amd64"],
True,
SHA,
"push",
"master",
[f"--platform linux/arm64,linux/amd64"],
),
], ],
) )
def test_get_docker_command( def test_get_docker_command(
build_preset, build_preset,
build_platform, build_platforms,
is_authenticated, is_authenticated,
sha, sha,
build_context, build_context,
@ -271,7 +281,7 @@ def test_get_docker_command(
): ):
cmd = docker_utils.get_docker_command( cmd = docker_utils.get_docker_command(
build_preset, build_preset,
build_platform, build_platforms,
is_authenticated, is_authenticated,
sha, sha,
build_context, build_context,