diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 3ba7b6bf42..73f4133848 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -2,7 +2,7 @@ name: Docker Publish Release on: release: - types: [published] + types: [published, edited] # Can be triggered manually workflow_dispatch: @@ -43,14 +43,6 @@ jobs: strategy: matrix: build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] - platform: ["linux/amd64", "linux/arm64"] - exclude: - # disabling because slow! no python wheels for arm/py39 and - # QEMU is slow! - - build_preset: "dev" - platform: "linux/arm64" - - build_preset: "lean" - platform: "linux/arm64" fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -88,8 +80,10 @@ jobs: EVENT="release" fi pip install click + # Make a multi-platform image ./scripts/build_docker.py \ ${{ matrix.build_preset }} \ "$EVENT" \ - --build_context_ref "$RELEASE" \ - --platform ${{ matrix.platform }} $FORCE_LATEST + --build_context_ref "$RELEASE" $FORCE_LATEST \ + --platform "linux/arm64" \ + --platform "linux/amd64" diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 8cc7966e00..8f804f741c 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -143,10 +143,9 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: superset-ci - SHA: ${{ steps.get-sha.outputs.sha }} IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci run: | - docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$SHA + docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci docker push -a $ECR_REGISTRY/$ECR_REPOSITORY ephemeral-env-up: @@ -181,7 +180,7 @@ jobs: aws ecr describe-images \ --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ --repository-name superset-ci \ - --image-ids imageTag=${{ steps.get-sha.outputs.sha }} + --image-ids imageTag=pr-${{ github.event.issue.number }}-ci - name: Fail on missing container image if: steps.check-image.outcome == 'failure' @@ -204,7 +203,7 @@ jobs: with: task-definition: .github/workflows/ecs-task-definition.json container-name: superset-ci - image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }} + image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci - name: Update env vars in the Amazon ECS task definition run: | diff --git a/scripts/build_docker.py b/scripts/build_docker.py index de6b8444f8..2a9e676d92 100755 --- a/scripts/build_docker.py +++ b/scripts/build_docker.py @@ -40,6 +40,10 @@ def run_cmd(command: str) -> str: output += line process.wait() # Wait for the subprocess to finish + + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command, output) + return output @@ -79,7 +83,7 @@ def make_docker_tag(l: list[str]) -> str: def get_docker_tags( build_preset: str, - build_platform: str, + build_platforms: list[str], sha: str, build_context: str, build_context_ref: str, @@ -91,17 +95,18 @@ def get_docker_tags( tags: set[str] = set() tag_chunks: list[str] = [] - short_build_platform = build_platform.replace("linux/", "").replace("64", "") - is_latest = is_latest_release(build_context_ref) if build_preset != "lean": # Always add the preset_build name if different from default (lean) tag_chunks += [build_preset] - if short_build_platform != "amd": - # Always a platform indicator if different from default (amd) - tag_chunks += [short_build_platform] + if len(build_platforms) == 1: + build_platform = build_platforms[0] + short_build_platform = build_platform.replace("linux/", "").replace("64", "") + if short_build_platform != "amd": + # Always a platform indicator if different from default (amd) + tag_chunks += [short_build_platform] # Always craft a tag for the SHA tags.add(make_docker_tag([sha] + tag_chunks)) @@ -123,7 +128,7 @@ def get_docker_tags( def get_docker_command( build_preset: str, - build_platform: str, + build_platforms: list[str], is_authenticated: bool, sha: str, build_context: str, @@ -160,7 +165,7 @@ def get_docker_command( tags = get_docker_tags( build_preset, - build_platform, + build_platforms, sha, build_context, build_context_ref, @@ -170,8 +175,14 @@ def get_docker_command( docker_args = "--load" if not is_authenticated else "--push" target_argument = f"--target {build_target}" if build_target else "" - short_build_platform = build_platform.replace("linux/", "").replace("64", "") - cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" + + cache_ref = f"{CACHE_REPO}:{py_ver}" + if len(build_platforms) == 1: + build_platform = build_platforms[0] + short_build_platform = build_platform.replace("linux/", "").replace("64", "") + cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" + platform_arg = "--platform " + ",".join(build_platforms) + cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}" cache_to_arg = ( f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else "" @@ -187,7 +198,7 @@ def get_docker_command( {cache_from_arg} \\ {cache_to_arg} \\ {build_arg} \\ - --platform {build_platform} \\ + {platform_arg} \\ --label sha={sha} \\ --label target={build_target} \\ --label build_trigger={build_context} \\ @@ -206,10 +217,12 @@ def get_docker_command( @click.option( "--platform", type=click.Choice(["linux/arm64", "linux/amd64"]), - default="linux/amd64", + default=["linux/amd64"], + multiple=True, ) @click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") +@click.option("--verbose", is_flag=True, help="Print more info") @click.option( "--force-latest", is_flag=True, help="Force the 'latest' tag on the release" ) @@ -217,9 +230,10 @@ def main( build_preset: str, build_context: str, build_context_ref: str, - platform: str, + platform: list[str], dry_run: bool, force_latest: bool, + verbose: bool, ) -> None: """ This script executes docker build and push commands based on given arguments. @@ -262,6 +276,8 @@ def main( """ ) script = script + docker_build_command + if verbose: + run_cmd("cat Dockerfile") stdout = run_cmd(script) else: print("Dry Run - Docker Build Command:") diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py index ee9ad66ead..1f3fd708f3 100644 --- a/tests/unit_tests/scripts/docker_build.py +++ b/tests/unit_tests/scripts/docker_build.py @@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool): @pytest.mark.parametrize( - "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags", + "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags", [ # PRs ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "pull_request", PR_ID, @@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool): ), ( "ci", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "pull_request", PR_ID, @@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool): # old releases ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "release", OLD_REL, @@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "release", OLD_REL, @@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "release", OLD_REL, @@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "release", OLD_REL, @@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool): # new releases ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "release", NEW_REL, @@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "release", NEW_REL, @@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "release", NEW_REL, @@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "release", NEW_REL, @@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool): # merge on master ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "push", "master", @@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "push", "master", @@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "push", "master", @@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "push", "master", @@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool): ], ) def test_get_docker_tags( - build_preset, build_platform, sha, build_context, build_context_ref, expected_tags + build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags ): tags = docker_utils.get_docker_tags( - build_preset, build_platform, sha, build_context, build_context_ref + build_preset, build_platforms, sha, build_context, build_context_ref ) for tag in expected_tags: assert tag in tags @pytest.mark.parametrize( - "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains", + "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains", [ ( "lean", - "linux/amd64", + ["linux/amd64"], True, SHA, "push", @@ -251,18 +251,28 @@ def test_get_docker_tags( ), ( "dev", - "linux/amd64", + ["linux/amd64"], False, SHA, "push", "master", ["--load", f"-t {REPO}:master-dev "], ), + # multi-platform + ( + "lean", + ["linux/arm64", "linux/amd64"], + True, + SHA, + "push", + "master", + [f"--platform linux/arm64,linux/amd64"], + ), ], ) def test_get_docker_command( build_preset, - build_platform, + build_platforms, is_authenticated, sha, build_context, @@ -271,7 +281,7 @@ def test_get_docker_command( ): cmd = docker_utils.get_docker_command( build_preset, - build_platform, + build_platforms, is_authenticated, sha, build_context,