From 2e2c9806b21b97f349b8180b5dbf72b0ad5c4e12 Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Fri, 3 Aug 2018 10:55:08 -0700 Subject: [PATCH] Reduce dashboard position_json data size (#5543) --- .../util/findFirstParentContainer_spec.js | 127 +++++++++--------- .../src/dashboard/components/Header.jsx | 26 +++- .../dashboard/containers/DashboardHeader.jsx | 7 +- .../src/dashboard/util/componentTypes.js | 24 ++-- .../assets/src/dashboard/util/constants.js | 10 +- superset/config.py | 1 + superset/migrations/versions/7fcdcde0761c_.py | 69 ++++++++++ superset/views/base.py | 1 + superset/views/core.py | 3 + 9 files changed, 185 insertions(+), 83 deletions(-) create mode 100644 superset/migrations/versions/7fcdcde0761c_.py diff --git a/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js b/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js index 4ab29bda06..ecaca67664 100644 --- a/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js +++ b/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js @@ -10,94 +10,91 @@ import { describe('findFirstParentContainer', () => { const mockGridLayout = { DASHBOARD_VERSION_KEY: 'v2', - DASHBOARD_ROOT_ID: { - type: 'DASHBOARD_ROOT_TYPE', - id: 'DASHBOARD_ROOT_ID', - children: ['DASHBOARD_GRID_ID'], + ROOT_ID: { + type: 'ROOT', + id: 'ROOT_ID', + children: ['GRID_ID'], }, - DASHBOARD_GRID_ID: { - type: 'DASHBOARD_GRID_TYPE', - id: 'DASHBOARD_GRID_ID', - children: ['DASHBOARD_ROW_TYPE-Bk45URrlQ'], + GRID_ID: { + type: 'GRID', + id: 'GRID_ID', + children: ['ROW-Bk45URrlQ'], }, - 'DASHBOARD_ROW_TYPE-Bk45URrlQ': { - type: 'DASHBOARD_ROW_TYPE', - id: 'DASHBOARD_ROW_TYPE-Bk45URrlQ', - children: ['DASHBOARD_CHART_TYPE-ryxVc8RHlX'], + 'ROW-Bk45URrlQ': { + type: 'ROW', + id: 'ROW-Bk45URrlQ', + children: ['CHART-ryxVc8RHlX'], }, - 'DASHBOARD_CHART_TYPE-ryxVc8RHlX': { - type: 'DASHBOARD_CHART_TYPE', - id: 'DASHBOARD_CHART_TYPE-ryxVc8RHlX', + 'CHART-ryxVc8RHlX': { + type: 'CHART', + id: 'CHART-ryxVc8RHlX', children: [], }, - DASHBOARD_HEADER_ID: { - id: 'DASHBOARD_HEADER_ID', - type: 'DASHBOARD_HEADER_TYPE', + HEADER_ID: { + id: 'HEADER_ID', + type: 'HEADER', }, }; const mockTabsLayout = { - 'DASHBOARD_CHART_TYPE-S1gilYABe7': { + 'CHART-S1gilYABe7': { children: [], - id: 'DASHBOARD_CHART_TYPE-S1gilYABe7', - type: 'DASHBOARD_CHART_TYPE', + id: 'CHART-S1gilYABe7', + type: 'CHART', }, - 'DASHBOARD_CHART_TYPE-SJli5K0HlQ': { + 'CHART-SJli5K0HlQ': { children: [], - id: 'DASHBOARD_CHART_TYPE-SJli5K0HlQ', - type: 'DASHBOARD_CHART_TYPE', + id: 'CHART-SJli5K0HlQ', + type: 'CHART', }, - DASHBOARD_GRID_ID: { + GRID_ID: { children: [], - id: 'DASHBOARD_GRID_ID', - type: 'DASHBOARD_GRID_TYPE', + id: 'GRID_ID', + type: 'GRID', }, - DASHBOARD_HEADER_ID: { - id: 'DASHBOARD_HEADER_ID', - type: 'DASHBOARD_HEADER_TYPE', + HEADER_ID: { + id: 'HEADER_ID', + type: 'HEADER', }, - DASHBOARD_ROOT_ID: { - children: ['DASHBOARD_TABS_TYPE-SkgJ5t0Bem'], - id: 'DASHBOARD_ROOT_ID', - type: 'DASHBOARD_ROOT_TYPE', + ROOT_ID: { + children: ['TABS-SkgJ5t0Bem'], + id: 'ROOT_ID', + type: 'ROOT', }, - 'DASHBOARD_ROW_TYPE-S1B8-JLgX': { - children: ['DASHBOARD_CHART_TYPE-SJli5K0HlQ'], - id: 'DASHBOARD_ROW_TYPE-S1B8-JLgX', - type: 'DASHBOARD_ROW_TYPE', + 'ROW-S1B8-JLgX': { + children: ['CHART-SJli5K0HlQ'], + id: 'ROW-S1B8-JLgX', + type: 'ROW', }, - 'DASHBOARD_ROW_TYPE-S1bUb1Ilm': { - children: ['DASHBOARD_CHART_TYPE-S1gilYABe7'], - id: 'DASHBOARD_ROW_TYPE-S1bUb1Ilm', - type: 'DASHBOARD_ROW_TYPE', + 'ROW-S1bUb1Ilm': { + children: ['CHART-S1gilYABe7'], + id: 'ROW-S1bUb1Ilm', + type: 'ROW', }, - 'DASHBOARD_TABS_TYPE-ByeLSWyLe7': { - children: ['DASHBOARD_TAB_TYPE-BJbLSZ1UeQ'], - id: 'DASHBOARD_TABS_TYPE-ByeLSWyLe7', - type: 'DASHBOARD_TABS_TYPE', + 'TABS-ByeLSWyLe7': { + children: ['TAB-BJbLSZ1UeQ'], + id: 'TABS-ByeLSWyLe7', + type: 'TABS', }, - 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem': { - children: [ - 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ', - 'DASHBOARD_TAB_TYPE-ByDBbkLlQ', - ], - id: 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem', + 'TABS-SkgJ5t0Bem': { + children: ['TAB-HkWJcFCHxQ', 'TAB-ByDBbkLlQ'], + id: 'TABS-SkgJ5t0Bem', meta: {}, - type: 'DASHBOARD_TABS_TYPE', + type: 'TABS', }, - 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ': { - children: ['DASHBOARD_ROW_TYPE-S1bUb1Ilm'], - id: 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ', - type: 'DASHBOARD_TAB_TYPE', + 'TAB-BJbLSZ1UeQ': { + children: ['ROW-S1bUb1Ilm'], + id: 'TAB-BJbLSZ1UeQ', + type: 'TAB', }, - 'DASHBOARD_TAB_TYPE-ByDBbkLlQ': { - children: ['DASHBOARD_ROW_TYPE-S1B8-JLgX'], - id: 'DASHBOARD_TAB_TYPE-ByDBbkLlQ', - type: 'DASHBOARD_TAB_TYPE', + 'TAB-ByDBbkLlQ': { + children: ['ROW-S1B8-JLgX'], + id: 'TAB-ByDBbkLlQ', + type: 'TAB', }, - 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ': { - children: ['DASHBOARD_TABS_TYPE-ByeLSWyLe7'], - id: 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ', - type: 'DASHBOARD_TAB_TYPE', + 'TAB-HkWJcFCHxQ': { + children: ['TABS-ByeLSWyLe7'], + id: 'TAB-HkWJcFCHxQ', + type: 'TAB', }, DASHBOARD_VERSION_KEY: 'v2', }; diff --git a/superset/assets/src/dashboard/components/Header.jsx b/superset/assets/src/dashboard/components/Header.jsx index 9f976cbab9..bc66a0136c 100644 --- a/superset/assets/src/dashboard/components/Header.jsx +++ b/superset/assets/src/dashboard/components/Header.jsx @@ -10,11 +10,16 @@ import UndoRedoKeylisteners from './UndoRedoKeylisteners'; import { chartPropShape } from '../util/propShapes'; import { t } from '../../locales'; -import { UNDO_LIMIT, SAVE_TYPE_OVERWRITE } from '../util/constants'; +import { + UNDO_LIMIT, + SAVE_TYPE_OVERWRITE, + DASHBOARD_POSITION_DATA_LIMIT, +} from '../util/constants'; const propTypes = { addSuccessToast: PropTypes.func.isRequired, addDangerToast: PropTypes.func.isRequired, + addWarningToast: PropTypes.func.isRequired, dashboardInfo: PropTypes.object.isRequired, dashboardTitle: PropTypes.string.isRequired, charts: PropTypes.objectOf(chartPropShape).isRequired, @@ -143,7 +148,24 @@ class Header extends React.PureComponent { default_filters: JSON.stringify(filters), }; - this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE); + // make sure positions data less than DB storage limitation: + const positionJSONLength = JSON.stringify(positions).length; + const limit = + dashboardInfo.common.conf.SUPERSET_DASHBOARD_POSITION_DATA_LIMIT || + DASHBOARD_POSITION_DATA_LIMIT; + if (positionJSONLength >= limit) { + this.props.addDangerToast( + t( + 'Your dashboard is too large. Please reduce the size before save it.', + ), + ); + } else { + if (positionJSONLength >= limit * 0.9) { + this.props.addWarningToast('Your dashboard is near the size limit.'); + } + + this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE); + } } render() { diff --git a/superset/assets/src/dashboard/containers/DashboardHeader.jsx b/superset/assets/src/dashboard/containers/DashboardHeader.jsx index 629b916456..3e54c62469 100644 --- a/superset/assets/src/dashboard/containers/DashboardHeader.jsx +++ b/superset/assets/src/dashboard/containers/DashboardHeader.jsx @@ -23,7 +23,11 @@ import { updateDashboardTitle, } from '../actions/dashboardLayout'; -import { addSuccessToast, addDangerToast } from '../../messageToasts/actions'; +import { + addSuccessToast, + addDangerToast, + addWarningToast, +} from '../../messageToasts/actions'; import { DASHBOARD_HEADER_ID } from '../util/constants'; @@ -59,6 +63,7 @@ function mapDispatchToProps(dispatch) { { addSuccessToast, addDangerToast, + addWarningToast, onUndo: undoLayoutAction, onRedo: redoLayoutAction, setEditMode, diff --git a/superset/assets/src/dashboard/util/componentTypes.js b/superset/assets/src/dashboard/util/componentTypes.js index b773417983..47478e6119 100644 --- a/superset/assets/src/dashboard/util/componentTypes.js +++ b/superset/assets/src/dashboard/util/componentTypes.js @@ -1,15 +1,15 @@ -export const CHART_TYPE = 'DASHBOARD_CHART_TYPE'; -export const COLUMN_TYPE = 'DASHBOARD_COLUMN_TYPE'; -export const DASHBOARD_HEADER_TYPE = 'DASHBOARD_HEADER_TYPE'; -export const DASHBOARD_GRID_TYPE = 'DASHBOARD_GRID_TYPE'; -export const DASHBOARD_ROOT_TYPE = 'DASHBOARD_ROOT_TYPE'; -export const DIVIDER_TYPE = 'DASHBOARD_DIVIDER_TYPE'; -export const HEADER_TYPE = 'DASHBOARD_HEADER_TYPE'; -export const MARKDOWN_TYPE = 'DASHBOARD_MARKDOWN_TYPE'; -export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE_TYPE'; -export const ROW_TYPE = 'DASHBOARD_ROW_TYPE'; -export const TABS_TYPE = 'DASHBOARD_TABS_TYPE'; -export const TAB_TYPE = 'DASHBOARD_TAB_TYPE'; +export const CHART_TYPE = 'CHART'; +export const COLUMN_TYPE = 'COLUMN'; +export const DASHBOARD_HEADER_TYPE = 'HEADER'; +export const DASHBOARD_GRID_TYPE = 'GRID'; +export const DASHBOARD_ROOT_TYPE = 'ROOT'; +export const DIVIDER_TYPE = 'DIVIDER'; +export const HEADER_TYPE = 'HEADER'; +export const MARKDOWN_TYPE = 'MARKDOWN'; +export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE'; +export const ROW_TYPE = 'ROW'; +export const TABS_TYPE = 'TABS'; +export const TAB_TYPE = 'TAB'; export default { CHART_TYPE, diff --git a/superset/assets/src/dashboard/util/constants.js b/superset/assets/src/dashboard/util/constants.js index 4fd5e400ec..b26cbff85f 100644 --- a/superset/assets/src/dashboard/util/constants.js +++ b/superset/assets/src/dashboard/util/constants.js @@ -1,7 +1,7 @@ // Ids -export const DASHBOARD_GRID_ID = 'DASHBOARD_GRID_ID'; -export const DASHBOARD_HEADER_ID = 'DASHBOARD_HEADER_ID'; -export const DASHBOARD_ROOT_ID = 'DASHBOARD_ROOT_ID'; +export const DASHBOARD_GRID_ID = 'GRID_ID'; +export const DASHBOARD_HEADER_ID = 'HEADER_ID'; +export const DASHBOARD_ROOT_ID = 'ROOT_ID'; export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY'; export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID'; @@ -40,3 +40,7 @@ export const UNDO_LIMIT = 50; // save dash options export const SAVE_TYPE_OVERWRITE = 'overwrite'; export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard'; + +// default dashboard layout data size limit +// could be overwritten by server-side config +export const DASHBOARD_POSITION_DATA_LIMIT = 65535; diff --git a/superset/config.py b/superset/config.py index 4e31358af7..ca9fcbdb6a 100644 --- a/superset/config.py +++ b/superset/config.py @@ -49,6 +49,7 @@ SUPERSET_CELERY_WORKERS = 32 # deprecated SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0' SUPERSET_WEBSERVER_PORT = 8088 SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated +SUPERSET_DASHBOARD_POSITION_DATA_LIMIT = 65535 EMAIL_NOTIFICATIONS = False CUSTOM_SECURITY_MANAGER = None SQLALCHEMY_TRACK_MODIFICATIONS = False diff --git a/superset/migrations/versions/7fcdcde0761c_.py b/superset/migrations/versions/7fcdcde0761c_.py new file mode 100644 index 0000000000..793e0fb68e --- /dev/null +++ b/superset/migrations/versions/7fcdcde0761c_.py @@ -0,0 +1,69 @@ +"""Reduce position_json size by remove extra space and component id prefix + +Revision ID: 7fcdcde0761c +Revises: c18bd4186f15 +Create Date: 2018-08-01 11:47:02.233971 + +""" + +# revision identifiers, used by Alembic. +import json +import re + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import ( + Table, Column, + Integer, String, Text, ForeignKey, +) +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +from superset import db + +revision = '7fcdcde0761c' +down_revision = 'c18bd4186f15' + +Base = declarative_base() + + +class Dashboard(Base): + """Declarative class to do query in upgrade""" + __tablename__ = 'dashboards' + id = sa.Column(sa.Integer, primary_key=True) + dashboard_title = sa.Column(String(500)) + position_json = sa.Column(sa.Text) + + +def is_v2_dash(positions): + return ( + isinstance(positions, dict) and + positions.get('DASHBOARD_VERSION_KEY') == 'v2' + ) + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + dashboards = session.query(Dashboard).all() + for i, dashboard in enumerate(dashboards): + original_text = dashboard.position_json or '' + position_json = json.loads(original_text or '{}') + if is_v2_dash(position_json): + # re-dump the json data and remove leading and trailing white spaces + text = json.dumps( + position_json, indent=None, separators=(',', ':'), sort_keys=True) + # remove DASHBOARD_ and _TYPE prefix/suffix in all the component ids + text = re.sub(r'DASHBOARD_(?!VERSION)', '', text) + text = text.replace('_TYPE', '') + + dashboard.position_json = text + print('dash id:{} position_json size from {} to {}' + .format(dashboard.id, len(original_text), len(text))) + session.merge(dashboard) + session.commit() + + +def downgrade(): + pass diff --git a/superset/views/base.py b/superset/views/base.py index 5d90284de3..8bcdee44f9 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -26,6 +26,7 @@ from superset.translations.utils import get_language_pack FRONTEND_CONF_KEYS = ( 'SUPERSET_WEBSERVER_TIMEOUT', + 'SUPERSET_DASHBOARD_POSITION_DATA_LIMIT', 'ENABLE_JAVASCRIPT_CONTROLS', ) diff --git a/superset/views/core.py b/superset/views/core.py index 93e79f3e39..667bfca13a 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -1643,6 +1643,9 @@ class Superset(BaseSupersetView): session.merge(slc) session.flush() + # remove leading and trailing white spaces in the dumped json + dashboard.position_json = json.dumps( + positions, indent=None, separators=(',', ':'), sort_keys=True) dashboard.position_json = json.dumps(positions, sort_keys=True) md = dashboard.params_dict dashboard.css = data.get('css')