feat: docker image tags documentation + tweaks (#26923)

Co-authored-by: Sam Firke <sfirke@users.noreply.github.com>
2024-02-01 18:04:23 -08:00 · 2024-02-01 18:04:23 -08:00 · 6b0d8ed265
parent fdff6d84f3
commit 6b0d8ed265
11 changed files with 145 additions and 39 deletions
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@ -1,8 +1,26 @@
-name: Docker
+name: Docker Publish Release

 on:
  release:
    types: [published]
+
+  # Can be triggered manually
+  workflow_dispatch:
+    inputs:
+      release:
+        required: true
+        description: The version to generate
+      git-ref:
+        required: true
+        description: The git reference to checkout prior to running the docker build
+      force-latest:
+        required: true
+        type: choice
+        default: 'false'
+        description: Whether to force a latest tag on the release
+        options:
+          - true
+          - false
 jobs:
  config:
    runs-on: "ubuntu-latest"
@ -53,6 +71,22 @@ jobs:
          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
        run: |
-          GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
+          RELEASE="${{ github.event.release.tag_name }}"
+          FORCE_LATEST=""
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            # in the case of a manually-triggered run, read release from input
+            RELEASE ="${{ github.event.inputs.release }}"
+            if [ "${{ github.event.inputs.force-latest }}" = "true" ]; then
+              FORCE_LATEST="--force-latest"
+            fi
+            # build_docker.py may not exist on that SHA, let's switcharoo to /tmp
+            cp ./scripts/build_docker.py /tmp
+            git checkout "${{ github.event.inputs.git-ref }}"
+            cp /tmp/build_docker.py scripts/
+          fi
          pip install click
-          ./scripts/build_docker.py ${{ matrix.build_preset }} release --platform ${{ matrix.platform }}
+          ./scripts/build_docker.py \
+            ${{ matrix.build_preset }} \
+            "${{ github.event_name }}" \
+            --build_context_ref "$RELEASE" \
+            --platform ${{ matrix.platform }} $FORCE_LATEST
--- a/.github/workflows/ephemeral-env.yml
+++ b/.github/workflows/ephemeral-env.yml
@ -122,13 +122,7 @@ jobs:

      - name: Build ephemeral env image
        run: |
-          docker buildx build --target ci \
-            --load \
-            -t ${{ steps.get-sha.outputs.sha }} \
-            -t "pr-${{ github.event.issue.number }}" \
-            --platform linux/amd64 \
-            --label "build_actor=${GITHUB_ACTOR}" \
-            .
+          ./scripts/build_docker.py "ci" "pull_request"

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
@ -147,7 +141,7 @@ jobs:
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          ECR_REPOSITORY: superset-ci
          SHA: ${{ steps.get-sha.outputs.sha }}
-          IMAGE_TAG: pr-${{ github.event.issue.number }}
+          IMAGE_TAG: pr-${{ github.event.issue.number }}-ci
        run: |
          docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
          docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$SHA
--- a/.github/workflows/no-op.yml
+++ b/.github/workflows/no-op.yml
@ -52,17 +52,3 @@ jobs:
        run: |
          echo "This is a no-op step for python-lint to ensure a successful status."
          exit 0
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for frontend-build
-        run: |
-          echo "This is a no-op step for frontend-build to ensure a successful status."
-          exit 0
-  docker-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for frontend-build
-        run: |
-          echo "This is a no-op step for frontend-build to ensure a successful status."
-          exit 0
--- a/docker-compose-non-dev.yml
+++ b/docker-compose-non-dev.yml
@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev}
+x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest}
 x-superset-depends-on: &superset-depends-on
  - db
  - redis
--- a/docs/README.md
+++ b/docs/README.md
@ -16,4 +16,7 @@ KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 -->
-This is the public documentation site for Superset, built using [Docusaurus 2](https://docusaurus.io/). See [CONTRIBUTING.md](../CONTRIBUTING.md#documentation) for documentation on contributing to documentation.
+This is the public documentation site for Superset, built using
+[Docusaurus 2](https://docusaurus.io/). See
+[CONTRIBUTING.md](../CONTRIBUTING.md#documentation) for documentation on
+contributing to documentation.
--- a/docs/docs/installation/docker.mdx
+++ b/docs/docs/installation/docker.mdx
@ -0,0 +1,66 @@
+# Docker Images and Tags
+
+The Apache Superset community extensively uses Docker for development, release,
+and productionizing Superset. This page details our Docker builds and tag naming
+schemes to help users navigate our offerings.
+
+Images are built and pushed to the [Superset Docker Hub repository](
+https://hub.docker.com/r/apache/superset). Different sets of images are created for:
+- Published releases, with tags like `3.0.0` and the `latest` tag.
+- Pull request iterations, each identified by tags starting with a SHA like
+  `8a2f7d378ab13c156fa183d9284b607ed69f5ecc`, and `pr-3454`, referencing the pull
+  request ID.
+- Merges to the main branch (`master`), resulting in new SHAs, with tags
+  prefixed with `master` for the latest `master` version.
+
+Each code version has multiple builds for different purposes, identified by suffixes:
+- **Build preset:** We offer various images for different needs:
+  - `lean`: The default Docker image, including both frontend and backend. Tags
+    without a build_preset are lean builds, e.g., `latest`.
+  - `dev`: For development, with a headless browser and root access.
+  - `py310`, e.g., Py310: Similar to lean but with a different Python version (in this example, 3.10).
+  - `ci`: For certain CI workloads.
+  - `websocket`: For Superset clusters supporting advanced features.
+  - `dockerize`: Used by Helm.
+- **Platform:** We build for `linux/arm64` and `linux/amd64`. The `-arm` suffix
+  indicates ARM builds (e.g., `latest-arm`), while tags without a suffix are for
+  AMD (e.g., `latest`).
+
+## Key Image Tags and Examples
+
+- `latest`: The latest official release build, implicitly the lean build on
+  `linux/amd64`.
+- `latest-dev`: the `-dev` image of the latest official release build, with a headless browser and root access.
+- `master`: The latest build from the `master` branch, implicitly lean on
+  `linux/amd64`.
+- `master-dev`: Similar to `master` but includes a headless browser and root access.
+- `pr-5252`: The latest commit in PR 5252.
+- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev-arm`: A `linux/arm64` build for
+  this specific SHA, which could be from a pull request, master merge, or release.
+- `30948dc-dev-arm`: Same as above, but SHA truncated to 7 characters for a
+  shorter handle on the same image
+- `websocket-latest`: The WebSocket image for use in a Superset cluster.
+
+For insights or modifications to the build matrix and tagging conventions,
+check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py)
+script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
+GitHub action.
+
+## Caching
+
+To accelerate builds, we follow Docker best practices and use `apache/superset-cache`.
+
+## On ARM builds and working with Apple silicon
+
+Apple's current generation of computers uses ARM-based CPUs, and Docker
+running on MACs seem to require `linux/arm64/v8` (at least one user's M2 was
+configured in that way). Setting the environment
+variable `DOCKER_DEFAULT_PLATFORM` to `linux/amd64` seems to function in
+term of leveraging, and building upon the Superset builds provided here.
+
+```
+export DOCKER_DEFAULT_PLATFORM=linux/amd64
+```
+
+Presumably, `linux/arm64/v8` would be more optimized for this generation
+of chips, but less compatible across the ARM ecosystem.
--- a/helm/superset/Chart.yaml
+++ b/helm/superset/Chart.yaml
@ -29,7 +29,7 @@ maintainers:
  - name: craig-rueda
    email: craig@craigrueda.com
    url: https://github.com/craig-rueda
-version: 0.12.1
+version: 0.12.2
 dependencies:
  - name: postgresql
    version: 12.1.6
--- a/helm/superset/README.md
+++ b/helm/superset/README.md
@ -23,7 +23,7 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs

 # superset

-![Version: 0.12.1](https://img.shields.io/badge/Version-0.12.1-informational?style=flat-square)
+![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square)

 Apache Superset is a modern, enterprise-ready business intelligence web application

@ -77,7 +77,7 @@ On helm this can be set on `extraSecretEnv.SUPERSET_SECRET_KEY` or `configOverri
 | hostAliases | list | `[]` | Custom hostAliases for all superset pods # https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"apachesuperset.docker.scarf.sh/apache/superset"` |  |
-| image.tag | string | `""` |  |
+| image.tag | string | `"latest"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |
--- a/helm/superset/values.yaml
+++ b/helm/superset/values.yaml
@ -179,7 +179,7 @@ extraConfigMountPath: "/app/configs"

 image:
  repository: apachesuperset.docker.scarf.sh/apache/superset
-  tag: ""
+  tag: "latest"
  pullPolicy: IfNotPresent

 imagePullSecrets: []
--- a/scripts/build_docker.py
+++ b/scripts/build_docker.py
@ -83,6 +83,7 @@ def get_docker_tags(
    sha: str,
    build_context: str,
    build_context_ref: str,
+    force_latest: bool = False,
 ) -> set[str]:
    """
    Return a set of tags given a given build context
@ -110,11 +111,13 @@ def get_docker_tags(
    if build_context == "release":
        # add a release tag
        tags.add(make_docker_tag([build_context_ref] + tag_chunks))
-        if is_latest:
+        if is_latest or force_latest:
            # add a latest tag
            tags.add(make_docker_tag(["latest"] + tag_chunks))
    elif build_context == "push" and build_context_ref == "master":
        tags.add(make_docker_tag(["master"] + tag_chunks))
+    elif build_context == "pull_request":
+        tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
    return tags


@ -125,6 +128,7 @@ def get_docker_command(
    sha: str,
    build_context: str,
    build_context_ref: str,
+    force_latest: bool = False,
 ) -> str:
    tag = ""
    build_target = ""
@ -160,6 +164,7 @@ def get_docker_command(
        sha,
        build_context,
        build_context_ref,
+        force_latest,
    )
    docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])

@ -205,12 +210,16 @@ def get_docker_command(
 )
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
+@click.option(
+    "--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
+)
 def main(
    build_preset: str,
    build_context: str,
    build_context_ref: str,
    platform: str,
    dry_run: bool,
+    force_latest: bool,
 ) -> None:
    """
    This script executes docker build and push commands based on given arguments.
@ -219,7 +228,16 @@ def main(
    is_authenticated = (
        True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
    )
-    build_context_ref = get_build_context_ref(build_context)
+
+    if force_latest and build_context != "release":
+        print(
+            "--force-latest can only be applied if the build context is set to 'release'"
+        )
+        exit(1)
+
+    if build_context == "release" and not build_context_ref.strip():
+        print("Release number has to be provided")
+        exit(1)

    docker_build_command = get_docker_command(
        build_preset,
@ -227,7 +245,8 @@ def main(
        is_authenticated,
        get_git_sha(),
        build_context,
-        get_build_context_ref(build_context),
+        build_context_ref,
+        force_latest,
    )

    if not dry_run:
--- a/tests/unit_tests/scripts/docker_build.py
+++ b/tests/unit_tests/scripts/docker_build.py
@ -65,7 +65,7 @@ def test_is_latest_release(release, expected_bool):
            SHA,
            "pull_request",
            PR_ID,
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"],
+            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"],
        ),
        (
            "ci",
@ -73,7 +73,7 @@ def test_is_latest_release(release, expected_bool):
            SHA,
            "pull_request",
            PR_ID,
-            [f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci"],
+            [f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"],
        ),
        (
            "lean",
@ -81,7 +81,7 @@ def test_is_latest_release(release, expected_bool):
            SHA,
            "pull_request",
            PR_ID,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}"],
+            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"],
        ),
        (
            "dev",
@ -89,7 +89,11 @@ def test_is_latest_release(release, expected_bool):
            SHA,
            "pull_request",
            PR_ID,
-            [f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"],
+            [
+                f"{REPO}:22e7c60-dev-arm",
+                f"{REPO}:{SHA}-dev-arm",
+                f"{REPO}:pr-{PR_ID}-dev-arm",
+            ],
        ),
        (
            "dev",
@ -97,7 +101,7 @@ def test_is_latest_release(release, expected_bool):
            SHA,
            "pull_request",
            PR_ID,
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"],
+            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"],
        ),
        # old releases
        (