mirror of https://github.com/apache/superset.git
Splitting up some of the Docker build steps (#8775)
* Splitting up some of the Docker build steps * Adding dev target which includes requirements-extra / Updating docker configs to handle async query running
This commit is contained in:
parent
665e94784c
commit
a44635e309
57
Dockerfile
57
Dockerfile
|
@ -15,6 +15,9 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
|
||||||
|
######################################################################
|
||||||
|
# PY stage that simply does a pip install on our requirements
|
||||||
|
######################################################################
|
||||||
ARG PY_VER=3.6.9
|
ARG PY_VER=3.6.9
|
||||||
FROM python:${PY_VER} AS superset-py
|
FROM python:${PY_VER} AS superset-py
|
||||||
|
|
||||||
|
@ -26,26 +29,37 @@ RUN mkdir /app \
|
||||||
libpq-dev \
|
libpq-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY ./requirements.txt ./docker/requirements-extra.txt ./setup.py ./MANIFEST.in ./README.md ./app/
|
# First, we just wanna install requirements, which will allow us to utilize the cache
|
||||||
COPY superset /app/superset
|
# in order to only build if and only if requirements change
|
||||||
|
COPY ./requirements.txt /app/
|
||||||
RUN cd /app \
|
RUN cd /app \
|
||||||
&& pip install -r requirements.txt -r requirements-extra.txt \
|
&& pip install --no-cache -r requirements.txt
|
||||||
&& pip install -e .
|
|
||||||
|
|
||||||
|
|
||||||
|
######################################################################
|
||||||
|
# Node stage to deal with static asset construction
|
||||||
|
######################################################################
|
||||||
FROM node:10-jessie AS superset-node
|
FROM node:10-jessie AS superset-node
|
||||||
|
|
||||||
COPY ./superset/assets /app/superset/assets
|
# NPM ci first, as to NOT invalidate previous steps except for when package.json changes
|
||||||
|
RUN mkdir -p /app/superset/assets
|
||||||
|
COPY ./superset/assets/package* /app/superset/assets/
|
||||||
|
RUN cd /app/superset/assets \
|
||||||
|
&& npm ci
|
||||||
|
|
||||||
|
# Next, copy in the rest and let webpack do its thing
|
||||||
|
COPY ./superset/assets /app/superset/assets
|
||||||
|
# This is BY FAR the most expensive step (thanks Terser!)
|
||||||
RUN cd /app/superset/assets \
|
RUN cd /app/superset/assets \
|
||||||
&& npm ci \
|
|
||||||
&& npm run build \
|
&& npm run build \
|
||||||
&& rm -rf node_modules
|
&& rm -rf node_modules
|
||||||
|
|
||||||
|
|
||||||
|
######################################################################
|
||||||
# Final lean image...
|
# Final lean image...
|
||||||
|
######################################################################
|
||||||
ARG PY_VER=3.6.9
|
ARG PY_VER=3.6.9
|
||||||
FROM python:${PY_VER}
|
FROM python:${PY_VER} AS lean
|
||||||
|
|
||||||
ENV LANG=C.UTF-8 \
|
ENV LANG=C.UTF-8 \
|
||||||
LC_ALL=C.UTF-8 \
|
LC_ALL=C.UTF-8 \
|
||||||
|
@ -57,7 +71,6 @@ ENV LANG=C.UTF-8 \
|
||||||
|
|
||||||
RUN useradd --user-group --no-create-home --no-log-init --shell /bin/bash superset \
|
RUN useradd --user-group --no-create-home --no-log-init --shell /bin/bash superset \
|
||||||
&& mkdir -p ${SUPERSET_HOME} ${PYTHONPATH} \
|
&& mkdir -p ${SUPERSET_HOME} ${PYTHONPATH} \
|
||||||
&& chown -R superset:superset /app \
|
|
||||||
&& apt-get update -y \
|
&& apt-get update -y \
|
||||||
&& apt-get install -y --no-install-recommends \
|
&& apt-get install -y --no-install-recommends \
|
||||||
build-essential \
|
build-essential \
|
||||||
|
@ -65,14 +78,18 @@ RUN useradd --user-group --no-create-home --no-log-init --shell /bin/bash supers
|
||||||
libpq-dev \
|
libpq-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY --from=superset-py --chown=superset:superset /app/superset /app/superset
|
|
||||||
COPY --from=superset-py /usr/local/lib/python3.6/site-packages/ /usr/local/lib/python3.6/site-packages/
|
COPY --from=superset-py /usr/local/lib/python3.6/site-packages/ /usr/local/lib/python3.6/site-packages/
|
||||||
# Copying site-packages doesn't move the CLIs, so let's copy them one by one
|
# Copying site-packages doesn't move the CLIs, so let's copy them one by one
|
||||||
COPY --from=superset-py /usr/local/bin/superset /usr/local/bin/gunicorn /usr/local/bin/celery /usr/local/bin/flask /usr/bin/
|
COPY --from=superset-py /usr/local/bin/gunicorn /usr/local/bin/celery /usr/local/bin/flask /usr/bin/
|
||||||
COPY --from=superset-py /app/apache_superset.egg-info /app/apache_superset.egg-info
|
|
||||||
|
|
||||||
COPY --from=superset-node /app/superset/assets /app/superset/assets
|
COPY --from=superset-node /app/superset/assets /app/superset/assets
|
||||||
|
|
||||||
|
## Lastly, let's install superset itself
|
||||||
|
COPY superset /app/superset
|
||||||
|
COPY setup.py MANIFEST.in README.md /app/
|
||||||
|
RUN cd /app \
|
||||||
|
&& chown -R superset:superset * \
|
||||||
|
&& pip install -e .
|
||||||
|
|
||||||
COPY ./docker/docker-entrypoint.sh /usr/bin/
|
COPY ./docker/docker-entrypoint.sh /usr/bin/
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
@ -84,3 +101,15 @@ HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"]
|
||||||
EXPOSE ${SUPERSET_PORT}
|
EXPOSE ${SUPERSET_PORT}
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||||
|
|
||||||
|
######################################################################
|
||||||
|
# Dev image...
|
||||||
|
######################################################################
|
||||||
|
FROM lean AS dev
|
||||||
|
|
||||||
|
COPY ./requirements-dev.txt ./docker/requirements-extra.txt /app/
|
||||||
|
|
||||||
|
USER root
|
||||||
|
RUN cd /app \
|
||||||
|
&& pip install --no-cache -r requirements-dev.txt -r requirements-extra.txt
|
||||||
|
USER superset
|
|
@ -17,6 +17,7 @@
|
||||||
x-superset-build: &superset-build
|
x-superset-build: &superset-build
|
||||||
context: ./
|
context: ./
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
|
target: dev
|
||||||
x-superset-depends-on: &superset-depends-on
|
x-superset-depends-on: &superset-depends-on
|
||||||
- postgres
|
- postgres
|
||||||
- redis
|
- redis
|
||||||
|
@ -25,6 +26,7 @@ x-superset-volumes: &superset-volumes
|
||||||
- ./docker/docker-init.sh:/app/docker-init.sh
|
- ./docker/docker-init.sh:/app/docker-init.sh
|
||||||
- ./docker/pythonpath_dev:/app/pythonpath
|
- ./docker/pythonpath_dev:/app/pythonpath
|
||||||
- ./superset:/app/superset
|
- ./superset:/app/superset
|
||||||
|
- superset_home:/app/superset_home
|
||||||
|
|
||||||
version: "3.7"
|
version: "3.7"
|
||||||
services:
|
services:
|
||||||
|
@ -78,6 +80,8 @@ services:
|
||||||
volumes: *superset-volumes
|
volumes: *superset-volumes
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
|
superset_home:
|
||||||
|
external: false
|
||||||
postgres:
|
postgres:
|
||||||
external: false
|
external: false
|
||||||
redis:
|
redis:
|
||||||
|
|
|
@ -15,26 +15,47 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
set -ex
|
set -e
|
||||||
|
|
||||||
|
STEP_CNT=4
|
||||||
|
|
||||||
|
echo_step() {
|
||||||
|
cat <<EOF
|
||||||
|
|
||||||
|
######################################################################
|
||||||
|
|
||||||
|
|
||||||
|
Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
|
||||||
|
|
||||||
|
|
||||||
|
######################################################################
|
||||||
|
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
# Create an admin user
|
# Create an admin user
|
||||||
echo "Setting up admin user..."
|
echo_step "1" "Starting" "Setting up admin user"
|
||||||
flask fab create-admin \
|
flask fab create-admin \
|
||||||
--username admin \
|
--username admin \
|
||||||
--firstname Superset \
|
--firstname Superset \
|
||||||
--lastname Admin \
|
--lastname Admin \
|
||||||
--email admin@superset.com \
|
--email admin@superset.com \
|
||||||
--password admin
|
--password admin
|
||||||
|
echo_step "1" "Complete" "Setting up admin user"
|
||||||
|
|
||||||
# Initialize the database
|
# Initialize the database
|
||||||
echo "Migrating the DB..."
|
echo_step "2" "Starting" "Applying DB migrations"
|
||||||
superset db upgrade
|
superset db upgrade
|
||||||
|
echo_step "2" "Complete" "Applying DB migrations"
|
||||||
|
|
||||||
if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
|
if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
|
||||||
# Load some data to play with
|
# Load some data to play with
|
||||||
|
echo_step "3" "Starting" "Loading examples"
|
||||||
superset load_examples
|
superset load_examples
|
||||||
|
echo_step "3" "Complete" "Loading examples"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Create default roles and permissions
|
# Create default roles and permissions
|
||||||
echo "Setting up roles and perms..."
|
echo_step "4" "Starting" "Setting up roles and perms"
|
||||||
superset init
|
superset init
|
||||||
|
echo_step "4" "Complete" "Setting up roles and perms"
|
||||||
|
|
|
@ -25,6 +25,8 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
from werkzeug.contrib.cache import FileSystemCache
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger()
|
logger = logging.getLogger()
|
||||||
|
|
||||||
|
@ -61,6 +63,8 @@ SQLALCHEMY_DATABASE_URI = "postgresql://%s:%s@%s:%s/%s" % (
|
||||||
REDIS_HOST = get_env_variable("REDIS_HOST")
|
REDIS_HOST = get_env_variable("REDIS_HOST")
|
||||||
REDIS_PORT = get_env_variable("REDIS_PORT")
|
REDIS_PORT = get_env_variable("REDIS_PORT")
|
||||||
|
|
||||||
|
RESULTS_BACKEND = FileSystemCache('/app/superset_home/sqllab')
|
||||||
|
|
||||||
|
|
||||||
class CeleryConfig(object):
|
class CeleryConfig(object):
|
||||||
BROKER_URL = "redis://%s:%s/0" % (REDIS_HOST, REDIS_PORT)
|
BROKER_URL = "redis://%s:%s/0" % (REDIS_HOST, REDIS_PORT)
|
||||||
|
|
Loading…
Reference in New Issue