Improve development experience with Docker (#5966)

- Improve Docker image
  - smaller
  - faster to build
  - deterministict dependencies (see #5958)
- Rework process to simplify setting things up
  - updated documentation
  - less commands to type
  - no files to move and modify
  - optional loading of samples
- Still working in standalone mode (without volumes for superset)

.dockerignore

@@ -0,0 +1,23 @@
+**/__pycache__/
+**/.mypy_cache
+**/.pytest_cache
+**/.tox
+**/.vscode
+**/.idea
+**/.coverage
+**/.DS_Store
+**/.eggs
+**/.python-version
+**/*.egg-info
+**/*.bak
+**/*.db
+**/*.pyc
+**/*.sqllite
+**/*.swp
+
+tests/
+docs/
+install/
+superset/assets/node_modules/
+superset/assets/cypress/
+superset/assets/coverage/

.gitignore

@@ -44,10 +44,3 @@ yarn-error.log
 *.iml
 venv
 @eaDir/
-
-# docker
-/Dockerfile
-/docker-build.sh
-/docker-compose.yml
-/docker-entrypoint.sh
-/docker-init.sh

CONTRIBUTING.md

@@ -255,11 +255,11 @@ Install third-party dependencies listed in `package.json`:
 # From the root of the repository
 cd superset/assets
 
-# Install yarn, a replacement for `npm install`
+# If needed, install yarn, a replacement for `npm install`
 npm install -g yarn
 
 # Install dependencies
-yarn install
+yarn
 ```
 
 Finally, to compile frontend assets, run any of the following commands.

contrib/docker/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=superset

contrib/docker/Dockerfile

@@ -1,64 +1,63 @@
 FROM python:3.6
 
-MAINTAINER Xiao Hanyu <hanyu.xiao@shopeemobile.com>
-
-# Add a normal user
-RUN useradd --user-group --create-home --shell /bin/bash work
+RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
 
 # Configure environment
 ENV LANG=C.UTF-8 \
-    LC_ALL=C.UTF-8 \
-    HOME=/home/work
+    LC_ALL=C.UTF-8
 
 RUN apt-get update -y
-#Install dependencies to fix `curl https support error` and `elaying package configuration warning`
+
+# Install dependencies to fix `curl https support error` and `elaying package configuration warning`
 RUN apt-get install -y apt-transport-https apt-utils
-# Install some dependencies
-# http://airbnb.io/superset/installation.html#os-dependencies
-RUN apt-get update -y && apt-get install -y build-essential libssl-dev \
+
+# Install superset dependencies
+# https://superset.incubator.apache.org/installation.html#os-dependencies
+RUN apt-get install -y build-essential libssl-dev \
     libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev
 
+# Install extra useful tool for development
 RUN apt-get install -y vim less postgresql-client redis-tools
 
 # Install nodejs for custom build
-# https://github.com/apache/incubator-superset/blob/master/docs/installation.rst#making-your-own-build
+# https://superset.incubator.apache.org/installation.html#making-your-own-build
 # https://nodejs.org/en/download/package-manager/
-RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
-RUN apt-get install -y nodejs
-RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -; \
-    echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list; \
-    apt-get update; \
-    apt-get install -y yarn
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+    && apt-get install -y nodejs
 
-RUN mkdir $HOME/incubator-superset
+# https://yarnpkg.com/lang/en/docs/install/#debian-stable
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && apt-get update \
+    && apt-get install -y yarn
 
-WORKDIR $HOME/incubator-superset
+WORKDIR /home/superset
 
-COPY ./ ./
+COPY requirements.txt .
+COPY requirements-dev.txt .
 
-RUN mkdir -p /home/work/.cache
-RUN pip install --upgrade setuptools pip
-RUN pip install -r requirements.txt
-RUN pip install -r requirements-dev.txt
-RUN pip install -e .
+RUN pip install --upgrade setuptools pip \
+    && pip install -r requirements.txt -r requirements-dev.txt \
+    && rm -rf /root/.cache/pip
 
-ENV PATH=/home/work/incubator-superset/superset/bin:$PATH \
-    PYTHONPATH=./superset/:$PYTHONPATH
+USER superset
 
-COPY docker-entrypoint.sh /usr/local/bin/
-RUN chmod +x /usr/local/bin/docker-entrypoint.sh
-RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat
+COPY --chown=superset:superset superset superset
 
-COPY ./superset ./superset
-RUN chown -R work:work $HOME
+ENV PATH=/home/superset/superset/bin:$PATH \
+    PYTHONPATH=/home/superset/superset/:$PYTHONPATH
 
-USER work
+RUN cd superset/assets \
+    && yarn --non-interactive --frozen-lockfile --link-duplicates \
+    && yarn run sync-backend \
+    && yarn run build \
+    && rm -rf node_modules \
+    && yarn cache clean
 
-RUN cd superset/assets && yarn
-RUN cd superset/assets && npm run build
+COPY contrib/docker/docker-init.sh .
+COPY contrib/docker/docker-entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
 
 HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"]
 
-ENTRYPOINT ["docker-entrypoint.sh"]
-
 EXPOSE 8088

contrib/docker/docker-build.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-docker build -t apache/incubator-superset -f Dockerfile .

contrib/docker/docker-compose.yml

@@ -2,14 +2,14 @@ version: '3'
 services:
   redis:
     image: redis:3.2
-    restart: always
+    restart: unless-stopped
     ports:
       - 6379:6379
     volumes:
       - redis:/data
   postgres:
     image: postgres:10
-    restart: always
+    restart: unless-stopped
     environment:
       POSTGRES_DB: superset
       POSTGRES_PASSWORD: superset
@@ -19,8 +19,10 @@ services:
     volumes:
       - postgres:/var/lib/postgresql/data
   superset:
-    image: apache/incubator-superset
-    restart: always
+    build:
+      context: ../../
+      dockerfile: contrib/docker/Dockerfile
+    restart: unless-stopped
     environment:
       POSTGRES_DB: superset
       POSTGRES_USER: superset
@@ -29,20 +31,21 @@ services:
       POSTGRES_PORT: 5432
       REDIS_HOST: redis
       REDIS_PORT: 6379
-      SUPERSET_ENV: local
+      # If using production, comment development volume below
+      #SUPERSET_ENV: production
+      SUPERSET_ENV: development
     ports:
       - 8088:8088
-    command: "tail -f /dev/null"
     depends_on:
       - postgres
       - redis
     volumes:
-      - .:/home/work/incubator-superset
-      - superset-node-modules:/home/work/incubator-superset/superset/assets/node_modules
+      # this is needed to communicate with the postgres and redis services
+      - ./superset_config.py:/home/superset/superset/superset_config.py
+      # this is needed for development, remove with SUPERSET_ENV=production
+      - ../../superset:/home/superset/superset
 volumes:
   postgres:
     external: false
   redis:
     external: false
-  superset-node-modules:
-    external: false

contrib/docker/docker-entrypoint.sh

@@ -3,9 +3,14 @@ set -ex
 
 if [ "$#" -ne 0 ]; then
     exec "$@"
-elif [ "$SUPERSET_ENV" = "local" ]; then
-    flask run -p 8088 --with-threads --reload --debugger  --host=0.0.0.0
+elif [ "$SUPERSET_ENV" = "development" ]; then
+    superset worker &
+    # needed by superset runserver
+    (cd superset/assets/ && yarn && yarn run sync-backend)
+    (cd superset/assets/ && yarn run dev) &
+    flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
 elif [ "$SUPERSET_ENV" = "production" ]; then
+    superset worker &
     superset runserver -a 0.0.0.0 -w $((2 * $(getconf _NPROCESSORS_ONLN) + 1))
 else
     superset --help

contrib/docker/docker-init.sh

@@ -8,17 +8,10 @@ fabmanager create-admin --app superset
 # Initialize the database
 superset db upgrade
 
-# Load some data to play with
-superset load_examples
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+    # Load some data to play with
+    superset load_examples
+fi
 
 # Create default roles and permissions
 superset init
-
-# Need to run `npm run build` when enter contains for first time
-cd superset/assets && npm run build && cd ../../
-
-# Start superset worker for SQL Lab
-superset worker &
-
-# Start the dev web server
-flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0

docs/installation.rst

@@ -43,22 +43,29 @@ If you know docker, then you're lucky, we have shortcut road for you to
 initialize development environment: ::
 
     git clone https://github.com/apache/incubator-superset/
-    cd incubator-superset
-    cp contrib/docker/{docker-build.sh,docker-compose.yml,docker-entrypoint.sh,docker-init.sh,Dockerfile} .
-    cp contrib/docker/superset_config.py superset/
-    bash -x docker-build.sh
-    docker-compose up -d
-    docker-compose exec superset bash
-    bash docker-init.sh
+    cd incubator-superset/contrib/docker
+    # prefix with SUPERSET_LOAD_EXAMPLES=yes to load examples:
+    docker-compose run --rm superset ./docker-init.sh
+    # you can run this command everytime you need to start superset now:
+    docker-compose up
 
 After several minutes for superset initialization to finish, you can open
 a browser and view `http://localhost:8088` to start your journey.
 
+From there, the container server will reload on modification of the superset python
+and javascript source code.
+Don't forget to reload the page to take the new frontend into account though.
+
+See also `CONTRIBUTING.md <https://github.com/apache/incubator-superset/blob/master/CONTRIBUTING.md#webpack-dev-server>`_,
+for alternative way of serving the frontend.
+
+It is also possible to run Superset in non-development mode: in the `docker-compose.yml` file remove
+the volumes needed for development and change the variable `SUPERSET_ENV` to `production`.
+
 If you are attempting to build on a Mac and it exits with 137 you need to increase your docker resources.
 OSX instructions: https://docs.docker.com/docker-for-mac/#advanced (Search for memory)
 
-Or if you're curious and want to install superset from bottom up, then go 
-ahead.
+Or if you're curious and want to install superset from bottom up, then go ahead.
 
 OS dependencies
 ---------------