pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Docker support (#8725) 

* Moving Docker stuff around

* docker-compose working as expected

* Updating README

* Addressing comments

* Utilizing requirements-extra
This commit is contained in:
Craig Rueda 2019-12-03 10:37:18 -08:00 committed by Daniel Vaz Gaspar
parent 69dcaca324
commit fce49680d7
16 changed files with 367 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@ -37,3 +37,4 @@ install/
superset/assets/node_modules/
superset/assets/cypress/
superset/assets/coverage/
venv

9
.gitignore vendored
View File

@ -46,7 +46,7 @@ env_py3
envpy3
env36
local_config.py
superset_config.py
/superset_config.py
superset.egg-info/
superset/bin/supersetc
tmp
@ -67,13 +67,6 @@ yarn-error.log
venv
@eaDir/
# Docker
./Dockerfile
./docker-build.sh
./docker-compose.yml
./docker-entrypoint.sh
./docker-init.sh
# Test data
celery_results.sqlite
celerybeat-schedule

4
CONTRIBUTING.md
View File

@ -456,6 +456,10 @@ If you run this service from somewhere other than your local machine, you may ne
npm install --global webpack webpack-cli webpack-dev-server
```
#### Docker
See docs [here](docker/README.md)
#### Updating NPM packages
Use npm in the prescribed way, making sure that

87
Dockerfile Normal file
View File

@ -0,0 +1,87 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
ARG PY_VER=3.6.9
FROM python:${PY_VER} AS superset-py
RUN mkdir /app \
&& apt-get update -y \
&& apt-get install -y --no-install-recommends \
build-essential \
default-libmysqlclient-dev \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
COPY ./requirements.txt ./docker/requirements-extra.txt ./setup.py ./MANIFEST.in ./README.md ./app/
COPY superset /app/superset
RUN cd /app \
&& pip install --upgrade setuptools pip \
&& pip install -r requirements.txt -r requirements-extra.txt \
&& pip install -e .
FROM node:10-jessie AS superset-node
COPY ./superset/assets /app/superset/assets
RUN cd /app/superset/assets \
&& npm ci \
&& npm run build \
&& rm -rf node_modules
# Final lean image...
ARG PY_VER=3.6.9
FROM python:${PY_VER}
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
FLASK_ENV=production \
FLASK_APP="superset.app:create_app()" \
PYTHONPATH="/app/pythonpath" \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8080
RUN useradd --user-group --no-create-home --no-log-init --shell /bin/bash superset \
&& mkdir -p ${SUPERSET_HOME} ${PYTHONPATH} \
&& chown -R superset:superset /app \
&& apt-get update -y \
&& apt-get install -y --no-install-recommends \
build-essential \
default-libmysqlclient-dev \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
COPY --from=superset-py --chown=superset:superset /app/superset /app/superset
COPY --from=superset-py /usr/local/lib/python3.6/site-packages/ /usr/local/lib/python3.6/site-packages/
# Copying site-packages doesn't move the CLIs, so let's copy them one by one
COPY --from=superset-py /usr/local/bin/superset /usr/local/bin/gunicorn /usr/local/bin/celery /usr/local/bin/flask /usr/bin/
COPY --from=superset-py /app/apache_superset.egg-info /app/apache_superset.egg-info
COPY --from=superset-node /app/superset/assets /app/superset/assets
COPY ./docker/docker-entrypoint.sh /usr/bin/
WORKDIR /app
USER superset
HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"]
EXPOSE ${SUPERSET_PORT}
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

72
contrib/docker/Dockerfile
View File

@ -1,72 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.6-jessie
RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
# Configure environment
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8
RUN apt-get update -y
# Install dependencies to fix `curl https support error` and `elaying package configuration warning`
RUN apt-get install -y apt-transport-https apt-utils
# Install superset dependencies
# https://superset.incubator.apache.org/installation.html#os-dependencies
RUN apt-get install -y build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev
# Install extra useful tool for development
RUN apt-get install -y vim less postgresql-client redis-tools
# Install nodejs for custom build
# https://superset.incubator.apache.org/installation.html#making-your-own-build
# https://nodejs.org/en/download/package-manager/
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
&& apt-get install -y nodejs
WORKDIR /home/superset
COPY requirements.txt .
COPY requirements-dev.txt .
COPY contrib/docker/requirements-extra.txt .
RUN pip --default-timeout=120 install --upgrade setuptools pip \
&& pip --default-timeout=120 install -r requirements.txt -r requirements-dev.txt -r requirements-extra.txt \
&& rm -rf /root/.cache/pip
COPY --chown=superset:superset superset superset
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/:$PYTHONPATH
USER superset
RUN cd superset/assets \
&& npm ci \
&& npm run build \
&& rm -rf node_modules
COPY contrib/docker/docker-init.sh .
COPY contrib/docker/docker-entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"]
EXPOSE 8088

71
contrib/docker/docker-compose.yml
View File

@ -1,71 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2'
services:
redis:
image: redis:3.2
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
volumes:
- redis:/data
postgres:
image: postgres:10
restart: unless-stopped
environment:
POSTGRES_DB: superset
POSTGRES_PASSWORD: superset
POSTGRES_USER: superset
ports:
- "127.0.0.1:5432:5432"
volumes:
- postgres:/var/lib/postgresql/data
superset:
build:
context: ../../
dockerfile: contrib/docker/Dockerfile
restart: unless-stopped
environment:
POSTGRES_DB: superset
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432
REDIS_HOST: redis
REDIS_PORT: 6379
# If using production, comment development volume below
#SUPERSET_ENV: production
SUPERSET_ENV: development
user: root:root
ports:
- 8088:8088
depends_on:
- postgres
- redis
volumes:
# this is needed to communicate with the postgres and redis services
- ./superset_config.py:/home/superset/superset/superset_config.py
# this is needed for development, remove with SUPERSET_ENV=production
- ../../superset:/home/superset/superset
volumes:
postgres:
external: false
redis:
external: false

58
contrib/docker/superset_config.py
View File

@ -1,58 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
def get_env_variable(var_name, default=None):
"""Get the environment variable or raise exception."""
try:
return os.environ[var_name]
except KeyError:
if default is not None:
return default
else:
error_msg = 'The environment variable {} was missing, abort...'\
.format(var_name)
raise EnvironmentError(error_msg)
POSTGRES_USER = get_env_variable('POSTGRES_USER')
POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD')
POSTGRES_HOST = get_env_variable('POSTGRES_HOST')
POSTGRES_PORT = get_env_variable('POSTGRES_PORT')
POSTGRES_DB = get_env_variable('POSTGRES_DB')
# The SQLAlchemy connection string.
SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % (POSTGRES_USER,
POSTGRES_PASSWORD,
POSTGRES_HOST,
POSTGRES_PORT,
POSTGRES_DB)
REDIS_HOST = get_env_variable('REDIS_HOST')
REDIS_PORT = get_env_variable('REDIS_PORT')
class CeleryConfig(object):
BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
CELERY_IMPORTS = ('superset.sql_lab', )
CELERY_RESULT_BACKEND = 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
CELERY_TASK_PROTOCOL = 1
CELERY_CONFIG = CeleryConfig

84
docker-compose.yml Normal file
View File

@ -0,0 +1,84 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
x-superset-build: &superset-build
context: ./
dockerfile: Dockerfile
x-superset-depends-on: &superset-depends-on
- postgres
- redis
x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker/docker-init.sh:/app/docker-init.sh
- ./docker/pythonpath_dev:/app/pythonpath
- ./superset:/app/superset
version: "3.7"
services:
redis:
image: redis:3.2
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
volumes:
- redis:/data
postgres:
env_file: docker/.env
image: postgres:10
restart: unless-stopped
ports:
- "127.0.0.1:5432:5432"
volumes:
- postgres:/var/lib/postgresql/data
superset:
build: *superset-build
command: ["flask", "run", "-p", "8088", "--with-threads", "--reload", "--debugger", "--host=0.0.0.0"]
env_file: docker/.env
restart: unless-stopped
ports:
- 8088:8088
depends_on: *superset-depends-on
volumes: *superset-volumes
superset-init:
build: *superset-build
command: ["/app/docker-init.sh"]
env_file: docker/.env
depends_on: *superset-depends-on
volumes: *superset-volumes
superset-node:
image: node:10-jessie
command: ["bash", "-c", "cd /app/superset/assets && npm install && npm run dev"]
env_file: docker/.env
depends_on: *superset-depends-on
volumes: *superset-volumes
superset-worker:
build: *superset-build
command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair"]
env_file: docker/.env
restart: unless-stopped
depends_on: *superset-depends-on
volumes: *superset-volumes
volumes:
postgres:
external: false
redis:
external: false

30
docker/.env Normal file
View File

@ -0,0 +1,30 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
COMPOSE_PROJECT_NAME=superset
POSTGRES_DB=superset
POSTGRES_HOST=postgres
POSTGRES_PASSWORD=superset
POSTGRES_PORT=5432
POSTGRES_USER=superset
# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
PYTHONPATH=/app/pythonpath:/app/pythonpath_docker
REDIS_HOST=redis
REDIS_PORT=6379
FLASK_ENV=development
SUPERSET_ENV=development
SUPERSET_LOAD_EXAMPLES=yes

27
contrib/docker/README.md → docker/README.md
View File

@ -19,17 +19,28 @@ under the License.
# Getting Started with Superset using Docker
Docker is an easy way to get started with Superset.
Docker is an easy way to get started with Superset.
## Prerequisites
1. Docker! [link](https://www.docker.com/get-started)
1. Docker-compose [link](https://docs.docker.com/compose/install/)
## Configuration
The `/app/pythonpath` folder is mounted from [./docker/pythonpath_dev](./docker/pythonpath_dev)
which contains a base configuration [./docker/pythonpath/superset_config.py](./docker/pythonpath/superset_config.py)
intended for use with local development.
### Local overrides
In order to override configuration settings locally, simply make a copy of [./docker/pythonpath/superset_config_local.example](./docker/pythonpath/superset_config_local.example)
into [./docker/pythonpath/superset_config_docker.py](./docker/pythonpath/superset_config_docker.py) (git ignored) and fill in your overrides.
## Initializing Database
To initialize the database with a user and example charts, dashboards and datasets run:
```bash
docker-compose run -e SUPERSET_LOAD_EXAMPLES=yes --rm superset ./docker-init.sh
```
This may take a minute.
The DB will initialize itself upon startup via the init container (superset-init)
(This may take a minute.)
## Normal Operation

23
contrib/docker/docker-entrypoint.sh → docker/docker-entrypoint.sh
View File

@ -15,24 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -ex
set -eo pipefail
if [ "$#" -ne 0 ]; then
exec "$@"
elif [ "$SUPERSET_ENV" = "development" ]; then
celery worker --app=superset.sql_lab:celery_app --pool=gevent -Ofair &
# needed by superset runserver
(cd superset/assets/ && npm ci)
(cd superset/assets/ && npm run dev) &
FLASK_ENV=development FLASK_APP="superset.app:create_app()" flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
elif [ "$SUPERSET_ENV" = "production" ]; then
celery worker --app=superset.sql_lab:celery_app --pool=gevent -Ofair &
exec gunicorn --bind 0.0.0.0:8088 \
--workers $((2 * $(getconf _NPROCESSORS_ONLN) + 1)) \
if [ "${#}" -ne 0 ]; then
exec "${@}"
else
gunicorn --bind "0.0.0.0:${SUPERSET_PORT}" \
--workers 1 \
--timeout 60 \
--limit-request-line 0 \
--limit-request-field_size 0 \
"superset.app:create_app()"
else
superset --help
"${FLASK_APP}"
fi

13
contrib/docker/docker-init.sh → docker/docker-init.sh
View File

@ -17,11 +17,17 @@
#
set -ex
# Create an admin user (you will be prompted to set username, first and last name before setting a password)
export FLASK_APP="superset.app:create_app()"
flask fab create-admin
# Create an admin user
echo "Setting up admin user..."
flask fab create-admin \
--username admin \
--firstname Superset \
--lastname Admin \
--email admin@superset.com \
--password admin
# Initialize the database
echo "Migrating the DB..."
superset db upgrade
if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
@ -30,4 +36,5 @@ if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
fi
# Create default roles and permissions
echo "Setting up roles and perms..."
superset init

8
contrib/docker/.env → docker/pythonpath_dev/.gitignore vendored
View File

@ -14,4 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
COMPOSE_PROJECT_NAME=superset
# Ignore everything
*
# DON'T ignore the .gitignore
!.gitignore
!superset_config.py
!superset_config_local.example

87
docker/pythonpath_dev/superset_config.py Normal file
View File

@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This file is included in the final Docker image and SHOULD be overridden when
# deploying the image to prod. Settings configured here are intended for use in local
# development environments. Also note that superset_config_docker.py is imported
# as a final step as a means to override "defaults" configured here
#
import logging
import os
logger = logging.getLogger()
def get_env_variable(var_name, default=None):
"""Get the environment variable or raise exception."""
try:
return os.environ[var_name]
except KeyError:
if default is not None:
return default
else:
error_msg = "The environment variable {} was missing, abort...".format(
var_name
)
raise EnvironmentError(error_msg)
POSTGRES_USER = get_env_variable("POSTGRES_USER")
POSTGRES_PASSWORD = get_env_variable("POSTGRES_PASSWORD")
POSTGRES_HOST = get_env_variable("POSTGRES_HOST")
POSTGRES_PORT = get_env_variable("POSTGRES_PORT")
POSTGRES_DB = get_env_variable("POSTGRES_DB")
# The SQLAlchemy connection string.
SQLALCHEMY_DATABASE_URI = "postgresql://%s:%s@%s:%s/%s" % (
POSTGRES_USER,
POSTGRES_PASSWORD,
POSTGRES_HOST,
POSTGRES_PORT,
POSTGRES_DB,
)
REDIS_HOST = get_env_variable("REDIS_HOST")
REDIS_PORT = get_env_variable("REDIS_PORT")
class CeleryConfig(object):
BROKER_URL = "redis://%s:%s/0" % (REDIS_HOST, REDIS_PORT)
CELERY_IMPORTS = ("superset.sql_lab",)
CELERY_RESULT_BACKEND = "redis://%s:%s/1" % (REDIS_HOST, REDIS_PORT)
CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}}
CELERY_TASK_PROTOCOL = 1
CELERY_CONFIG = CeleryConfig
#
# Optionally import superset_config_docker.py (which will have been included on
# the PYTHONPATH) in order to allow for local settings to be overridden
#
try:
from superset_config_docker import * # noqa
import superset_config_docker
logger.info(
f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"
)
except ImportError:
logger.info("Using default Docker config...")

27
docker/pythonpath_dev/superset_config_local.example Normal file
View File

@ -0,0 +1,27 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# This is an example "local" configuration file. In order to set/override config
# options that ONLY apply to your local environment, simply copy/rename this file
# to docker/pythonpath/superset_config_docker.py
# It ends up being imported by docker/superset_config.py which is loaded by
# superset/config.py
#
SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://pguser:pgpwd@some.host/superset"
SQLALCHEMY_ECHO = True

4
contrib/docker/requirements-extra.txt → docker/requirements-extra.txt
View File

@ -14,4 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
gevent==1.4.0
gevent==1.4.0
psycopg2-binary==2.7.5
redis==3.2.1