Init docker for local development environment. (#4193)

This commit will try to dockerize superset in local development
environment.

The basic design is:
- Enable superset, redis and postgres service instead of using sqlite,
  just want to simulate production environment settings
- Use environment variables to config various app settings. It's easy to
  run and config superset to any environment if we use environment than
  traditional config files
- For local development environment, we just expose postgres and redis
  to local host machine thus you can connect local port via `psql` or
  `redis-cli`
- Wrap start up command in a standard `docker-entrypoint.sh`, and use
  `tail -f /dev/null` combined with manually `superset runserver -d` to
  make sure that code error didn't cause the container to fail.
- Use volumes to share code between host and container, thus you can use
  your favourite tools to modify code and your code will run in
  containerized environment
- Use volumes to persistent postgres and redis data, and also
  `node_modules` data.
  - If we don't cache `node_modules` in docker volume, then every time
    run docker build, the `node_modules` directory, will is about 500 MB
    large, will be sent to docker daemon, and make the build quite slow.
- Wrap initialization commands to a single script `docker-init.sh`

After this dockerize setup, any developers who want to contribute to
superset, just follow three easy steps:

```
git clone https://github.com/apache/incubator-superset/
cd incubator-superset
cp contrib/docker/{docker-build.sh,docker-compose.yml,docker-entrypoint.sh,docker-init.sh,Dockerfile} .
cp contrib/docker/superset_config.py superset/
bash -x docker-build.sh
docker-compose up -d
docker-compose exec superset bash
bash docker-init.sh
```

.gitignore

@@ -41,3 +41,10 @@ superset/assets/version_info.json
 *.iml
 venv
 @eaDir/
+
+# docker
+/Dockerfile
+/docker-build.sh
+/docker-compose.yml
+/docker-entrypoint.sh
+/docker-init.sh

contrib/docker/Dockerfile

@@ -0,0 +1,60 @@
+FROM python:3.6
+
+MAINTAINER Xiao Hanyu <hanyu.xiao@shopeemobile.com>
+
+# Add a normal user
+RUN useradd --user-group --create-home --shell /bin/bash work
+
+# Configure environment
+ENV LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    HOME=/home/work
+
+RUN apt-get update -y
+
+# Install some dependencies
+# http://airbnb.io/superset/installation.html#os-dependencies
+RUN apt-get update -y && apt-get install -y build-essential libssl-dev \
+    libffi-dev python3-dev libsasl2-dev libldap2-dev
+
+RUN apt-get install -y vim less postgresql-client redis-tools
+
+# Install nodejs for custom build
+# https://github.com/apache/incubator-superset/blob/master/docs/installation.rst#making-your-own-build
+# https://nodejs.org/en/download/package-manager/
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+RUN apt-get install -y nodejs
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -; \
+    echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list; \
+    apt-get update; \
+    apt-get install -y yarn
+
+RUN mkdir $HOME/incubator-superset
+
+WORKDIR $HOME/incubator-superset
+
+COPY ./ ./
+
+RUN pip install --upgrade setuptools pip
+RUN pip install -e . && pip install -r requirements-dev.txt
+
+ENV PATH=/home/work/incubator-superset/superset/bin:$PATH \
+    PYTHONPATH=./superset/:$PYTHONPATH
+
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat
+
+COPY ./superset ./superset
+RUN chown -R work:work $HOME
+
+USER work
+
+RUN cd superset/assets && yarn
+RUN cd superset/assets && npm run build
+
+HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"]
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+EXPOSE 8088

contrib/docker/docker-build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -ex
+
+docker build -t apache/incubator-superset -f Dockerfile .

contrib/docker/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3'
+services:
+  redis:
+    image: redis:3.2
+    restart: always
+    ports:
+      - 6379:6379
+    volumes:
+      - redis:/data
+  postgres:
+    image: postgres:10
+    restart: always
+    environment:
+      POSTGRES_DB: superset
+      POSTGRES_PASSWORD: superset
+      POSTGRES_USER: superset
+    ports:
+      - 5432:5432
+    volumes:
+      - postgres:/var/lib/postgresql/data
+  superset:
+    image: apache/incubator-superset
+    restart: always
+    environment:
+      POSTGRES_DB: superset
+      POSTGRES_USER: superset
+      POSTGRES_PASSWORD: superset
+      POSTGRES_HOST: postgres
+      POSTGRES_PORT: 5432
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      SUPERSET_ENV: local
+    ports:
+      - 8088:8088
+    command: "tail -f /dev/null"
+    depends_on:
+      - postgres
+      - redis
+    volumes:
+      - .:/home/work/incubator-superset
+      - superset-node-modules:/home/work/incubator-superset/superset/assets/node_modules
+volumes:
+  postgres:
+    external: false
+  redis:
+    external: false
+  superset-node-modules:
+    external: false

contrib/docker/docker-entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+if [ "$#" -ne 0 ]; then
+    exec "$@"
+elif [ "$SUPERSET_ENV" = "local" ]; then
+    superset runserver -d
+elif [ "$SUPERSET_ENV" = "production" ]; then
+    superset runserver -a 0.0.0.0 -w $((2 * $(getconf _NPROCESSORS_ONLN) + 1))
+else
+    superset --help
+fi

contrib/docker/docker-init.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Create an admin user (you will be prompted to set username, first and last name before setting a password)
+fabmanager create-admin --app superset
+
+# Initialize the database
+superset db upgrade
+
+# Load some data to play with
+superset load_examples
+
+# Create default roles and permissions
+superset init
+
+# Need to run `npm run build` when enter contains for first time
+cd superset/assets && npm run build && cd ../../
+
+# Start superset worker for SQL Lab
+superset worker &
+
+# To start a development web server, use the -d switch
+superset runserver -d

contrib/docker/superset_config.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+
+
+def get_env_variable(var_name, default=None):
+    """Get the environment variable or raise exception."""
+    try:
+        return os.environ[var_name]
+    except KeyError:
+        if default is not None:
+            return default
+        else:
+            error_msg = 'The environment variable {} was missing, abort...'\
+                        .format(var_name)
+            raise EnvironmentError(error_msg)
+
+
+POSTGRES_USER = get_env_variable('POSTGRES_USER')
+POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD')
+POSTGRES_HOST = get_env_variable('POSTGRES_HOST')
+POSTGRES_PORT = get_env_variable('POSTGRES_PORT')
+POSTGRES_DB = get_env_variable('POSTGRES_DB')
+
+# The SQLAlchemy connection string.
+SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % (POSTGRES_USER,
+                                                           POSTGRES_PASSWORD,
+                                                           POSTGRES_HOST,
+                                                           POSTGRES_PORT,
+                                                           POSTGRES_DB)
+
+REDIS_HOST = get_env_variable('REDIS_HOST')
+REDIS_PORT = get_env_variable('REDIS_PORT')
+
+
+class CeleryConfig(object):
+    BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
+    CELERY_IMPORTS = ('superset.sql_lab', )
+    CELERY_RESULT_BACKEND = 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
+    CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
+    CELERY_TASK_PROTOCOL = 1
+
+
+CELERY_CONFIG = CeleryConfig

docs/installation.rst

@@ -35,6 +35,27 @@ The Superset web server and the Superset Celery workers (optional)
 are stateless, so you can scale out by running on as many servers
 as needed.
 
+Start with Docker
+-----------------
+
+If you know docker, then you're lucky, we have shortcut road for you to 
+initialize development environment: ::
+
+    git clone https://github.com/apache/incubator-superset/
+    cd incubator-superset
+    cp contrib/docker/{docker-build.sh,docker-compose.yml,docker-entrypoint.sh,docker-init.sh,Dockerfile} .
+    cp contrib/docker/superset_config.py superset/
+    bash -x docker-build.sh
+    docker-compose up -d
+    docker-compose exec superset bash
+    bash docker-init.sh
+
+After several minutes for sueprset initialization to finish, you can open a 
+a browser and view `http://localhost:8088` to start your journey.
+
+Or if you're curious and want to install superset from bottom up, then go 
+ahead.
+
 OS dependencies
 ---------------