pt/superset
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2024-09-12 08:39:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Reduce dashboard position_json data size (#5543)

Browse Source
This commit is contained in:
Grace Guo 2018-08-03 10:55:08 -07:00 committed by GitHub
parent e1f4db8e24
commit 2e2c9806b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 185 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
View File

@ -10,94 +10,91 @@ import {
describe('findFirstParentContainer', () => { describe('findFirstParentContainer', () => {
const mockGridLayout = { const mockGridLayout = {
DASHBOARD_VERSION_KEY: 'v2', DASHBOARD_VERSION_KEY: 'v2',
DASHBOARD_ROOT_ID: { ROOT_ID: {
type: 'DASHBOARD_ROOT_TYPE', type: 'ROOT',
id: 'DASHBOARD_ROOT_ID', id: 'ROOT_ID',
children: ['DASHBOARD_GRID_ID'], children: ['GRID_ID'],
}, },
DASHBOARD_GRID_ID: { GRID_ID: {
type: 'DASHBOARD_GRID_TYPE', type: 'GRID',
id: 'DASHBOARD_GRID_ID', id: 'GRID_ID',
children: ['DASHBOARD_ROW_TYPE-Bk45URrlQ'], children: ['ROW-Bk45URrlQ'],
}, },
'DASHBOARD_ROW_TYPE-Bk45URrlQ': { 'ROW-Bk45URrlQ': {
type: 'DASHBOARD_ROW_TYPE', type: 'ROW',
id: 'DASHBOARD_ROW_TYPE-Bk45URrlQ', id: 'ROW-Bk45URrlQ',
children: ['DASHBOARD_CHART_TYPE-ryxVc8RHlX'], children: ['CHART-ryxVc8RHlX'],
}, },
'DASHBOARD_CHART_TYPE-ryxVc8RHlX': { 'CHART-ryxVc8RHlX': {
type: 'DASHBOARD_CHART_TYPE', type: 'CHART',
id: 'DASHBOARD_CHART_TYPE-ryxVc8RHlX', id: 'CHART-ryxVc8RHlX',
children: [], children: [],
}, },
DASHBOARD_HEADER_ID: { HEADER_ID: {
id: 'DASHBOARD_HEADER_ID', id: 'HEADER_ID',
type: 'DASHBOARD_HEADER_TYPE', type: 'HEADER',
}, },
}; };
const mockTabsLayout = { const mockTabsLayout = {
'DASHBOARD_CHART_TYPE-S1gilYABe7': { 'CHART-S1gilYABe7': {
children: [], children: [],
id: 'DASHBOARD_CHART_TYPE-S1gilYABe7', id: 'CHART-S1gilYABe7',
type: 'DASHBOARD_CHART_TYPE', type: 'CHART',
}, },
'DASHBOARD_CHART_TYPE-SJli5K0HlQ': { 'CHART-SJli5K0HlQ': {
children: [], children: [],
id: 'DASHBOARD_CHART_TYPE-SJli5K0HlQ', id: 'CHART-SJli5K0HlQ',
type: 'DASHBOARD_CHART_TYPE', type: 'CHART',
}, },
DASHBOARD_GRID_ID: { GRID_ID: {
children: [], children: [],
id: 'DASHBOARD_GRID_ID', id: 'GRID_ID',
type: 'DASHBOARD_GRID_TYPE', type: 'GRID',
}, },
DASHBOARD_HEADER_ID: { HEADER_ID: {
id: 'DASHBOARD_HEADER_ID', id: 'HEADER_ID',
type: 'DASHBOARD_HEADER_TYPE', type: 'HEADER',
}, },
DASHBOARD_ROOT_ID: { ROOT_ID: {
children: ['DASHBOARD_TABS_TYPE-SkgJ5t0Bem'], children: ['TABS-SkgJ5t0Bem'],
id: 'DASHBOARD_ROOT_ID', id: 'ROOT_ID',
type: 'DASHBOARD_ROOT_TYPE', type: 'ROOT',
}, },
'DASHBOARD_ROW_TYPE-S1B8-JLgX': { 'ROW-S1B8-JLgX': {
children: ['DASHBOARD_CHART_TYPE-SJli5K0HlQ'], children: ['CHART-SJli5K0HlQ'],
id: 'DASHBOARD_ROW_TYPE-S1B8-JLgX', id: 'ROW-S1B8-JLgX',
type: 'DASHBOARD_ROW_TYPE', type: 'ROW',
}, },
'DASHBOARD_ROW_TYPE-S1bUb1Ilm': { 'ROW-S1bUb1Ilm': {
children: ['DASHBOARD_CHART_TYPE-S1gilYABe7'], children: ['CHART-S1gilYABe7'],
id: 'DASHBOARD_ROW_TYPE-S1bUb1Ilm', id: 'ROW-S1bUb1Ilm',
type: 'DASHBOARD_ROW_TYPE', type: 'ROW',
}, },
'DASHBOARD_TABS_TYPE-ByeLSWyLe7': { 'TABS-ByeLSWyLe7': {
children: ['DASHBOARD_TAB_TYPE-BJbLSZ1UeQ'], children: ['TAB-BJbLSZ1UeQ'],
id: 'DASHBOARD_TABS_TYPE-ByeLSWyLe7', id: 'TABS-ByeLSWyLe7',
type: 'DASHBOARD_TABS_TYPE', type: 'TABS',
}, },
'DASHBOARD_TABS_TYPE-SkgJ5t0Bem': { 'TABS-SkgJ5t0Bem': {
children: [ children: ['TAB-HkWJcFCHxQ', 'TAB-ByDBbkLlQ'],
'DASHBOARD_TAB_TYPE-HkWJcFCHxQ', id: 'TABS-SkgJ5t0Bem',
'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
],
id: 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem',
meta: {}, meta: {},
type: 'DASHBOARD_TABS_TYPE', type: 'TABS',
}, },
'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ': { 'TAB-BJbLSZ1UeQ': {
children: ['DASHBOARD_ROW_TYPE-S1bUb1Ilm'], children: ['ROW-S1bUb1Ilm'],
id: 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ', id: 'TAB-BJbLSZ1UeQ',
type: 'DASHBOARD_TAB_TYPE', type: 'TAB',
}, },
'DASHBOARD_TAB_TYPE-ByDBbkLlQ': { 'TAB-ByDBbkLlQ': {
children: ['DASHBOARD_ROW_TYPE-S1B8-JLgX'], children: ['ROW-S1B8-JLgX'],
id: 'DASHBOARD_TAB_TYPE-ByDBbkLlQ', id: 'TAB-ByDBbkLlQ',
type: 'DASHBOARD_TAB_TYPE', type: 'TAB',
}, },
'DASHBOARD_TAB_TYPE-HkWJcFCHxQ': { 'TAB-HkWJcFCHxQ': {
children: ['DASHBOARD_TABS_TYPE-ByeLSWyLe7'], children: ['TABS-ByeLSWyLe7'],
id: 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ', id: 'TAB-HkWJcFCHxQ',
type: 'DASHBOARD_TAB_TYPE', type: 'TAB',
}, },
DASHBOARD_VERSION_KEY: 'v2', DASHBOARD_VERSION_KEY: 'v2',
}; };

26
superset/assets/src/dashboard/components/Header.jsx
View File

@ -10,11 +10,16 @@ import UndoRedoKeylisteners from './UndoRedoKeylisteners';
import { chartPropShape } from '../util/propShapes'; import { chartPropShape } from '../util/propShapes';
import { t } from '../../locales'; import { t } from '../../locales';
import { UNDO_LIMIT, SAVE_TYPE_OVERWRITE } from '../util/constants'; import {
UNDO_LIMIT,
SAVE_TYPE_OVERWRITE,
DASHBOARD_POSITION_DATA_LIMIT,
} from '../util/constants';
const propTypes = { const propTypes = {
addSuccessToast: PropTypes.func.isRequired, addSuccessToast: PropTypes.func.isRequired,
addDangerToast: PropTypes.func.isRequired, addDangerToast: PropTypes.func.isRequired,
addWarningToast: PropTypes.func.isRequired,
dashboardInfo: PropTypes.object.isRequired, dashboardInfo: PropTypes.object.isRequired,
dashboardTitle: PropTypes.string.isRequired, dashboardTitle: PropTypes.string.isRequired,
charts: PropTypes.objectOf(chartPropShape).isRequired, charts: PropTypes.objectOf(chartPropShape).isRequired,
@ -143,7 +148,24 @@ class Header extends React.PureComponent {
default_filters: JSON.stringify(filters), default_filters: JSON.stringify(filters),
}; };
this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE); // make sure positions data less than DB storage limitation:
const positionJSONLength = JSON.stringify(positions).length;
const limit =
dashboardInfo.common.conf.SUPERSET_DASHBOARD_POSITION_DATA_LIMIT ||
DASHBOARD_POSITION_DATA_LIMIT;
if (positionJSONLength >= limit) {
this.props.addDangerToast(
t(
'Your dashboard is too large. Please reduce the size before save it.',
),
);
} else {
if (positionJSONLength >= limit * 0.9) {
this.props.addWarningToast('Your dashboard is near the size limit.');
}
this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
}
} }
render() { render() {

7
superset/assets/src/dashboard/containers/DashboardHeader.jsx
View File

@ -23,7 +23,11 @@ import {
updateDashboardTitle, updateDashboardTitle,
} from '../actions/dashboardLayout'; } from '../actions/dashboardLayout';
import { addSuccessToast, addDangerToast } from '../../messageToasts/actions'; import {
addSuccessToast,
addDangerToast,
addWarningToast,
} from '../../messageToasts/actions';
import { DASHBOARD_HEADER_ID } from '../util/constants'; import { DASHBOARD_HEADER_ID } from '../util/constants';
@ -59,6 +63,7 @@ function mapDispatchToProps(dispatch) {
{ {
addSuccessToast, addSuccessToast,
addDangerToast, addDangerToast,
addWarningToast,
onUndo: undoLayoutAction, onUndo: undoLayoutAction,
onRedo: redoLayoutAction, onRedo: redoLayoutAction,
setEditMode, setEditMode,

24
superset/assets/src/dashboard/util/componentTypes.js
View File

@ -1,15 +1,15 @@
export const CHART_TYPE = 'DASHBOARD_CHART_TYPE'; export const CHART_TYPE = 'CHART';
export const COLUMN_TYPE = 'DASHBOARD_COLUMN_TYPE'; export const COLUMN_TYPE = 'COLUMN';
export const DASHBOARD_HEADER_TYPE = 'DASHBOARD_HEADER_TYPE'; export const DASHBOARD_HEADER_TYPE = 'HEADER';
export const DASHBOARD_GRID_TYPE = 'DASHBOARD_GRID_TYPE'; export const DASHBOARD_GRID_TYPE = 'GRID';
export const DASHBOARD_ROOT_TYPE = 'DASHBOARD_ROOT_TYPE'; export const DASHBOARD_ROOT_TYPE = 'ROOT';
export const DIVIDER_TYPE = 'DASHBOARD_DIVIDER_TYPE'; export const DIVIDER_TYPE = 'DIVIDER';
export const HEADER_TYPE = 'DASHBOARD_HEADER_TYPE'; export const HEADER_TYPE = 'HEADER';
export const MARKDOWN_TYPE = 'DASHBOARD_MARKDOWN_TYPE'; export const MARKDOWN_TYPE = 'MARKDOWN';
export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE_TYPE'; export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE';
export const ROW_TYPE = 'DASHBOARD_ROW_TYPE'; export const ROW_TYPE = 'ROW';
export const TABS_TYPE = 'DASHBOARD_TABS_TYPE'; export const TABS_TYPE = 'TABS';
export const TAB_TYPE = 'DASHBOARD_TAB_TYPE'; export const TAB_TYPE = 'TAB';
export default { export default {
CHART_TYPE, CHART_TYPE,

10
superset/assets/src/dashboard/util/constants.js
View File

@ -1,7 +1,7 @@
// Ids // Ids
export const DASHBOARD_GRID_ID = 'DASHBOARD_GRID_ID'; export const DASHBOARD_GRID_ID = 'GRID_ID';
export const DASHBOARD_HEADER_ID = 'DASHBOARD_HEADER_ID'; export const DASHBOARD_HEADER_ID = 'HEADER_ID';
export const DASHBOARD_ROOT_ID = 'DASHBOARD_ROOT_ID'; export const DASHBOARD_ROOT_ID = 'ROOT_ID';
export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY'; export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY';
export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID'; export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID';
@ -40,3 +40,7 @@ export const UNDO_LIMIT = 50;
// save dash options // save dash options
export const SAVE_TYPE_OVERWRITE = 'overwrite'; export const SAVE_TYPE_OVERWRITE = 'overwrite';
export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard'; export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard';
// default dashboard layout data size limit
// could be overwritten by server-side config
export const DASHBOARD_POSITION_DATA_LIMIT = 65535;

1
superset/config.py
View File

@ -49,6 +49,7 @@ SUPERSET_CELERY_WORKERS = 32 # deprecated
SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0' SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
SUPERSET_WEBSERVER_PORT = 8088 SUPERSET_WEBSERVER_PORT = 8088
SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated
SUPERSET_DASHBOARD_POSITION_DATA_LIMIT = 65535
EMAIL_NOTIFICATIONS = False EMAIL_NOTIFICATIONS = False
CUSTOM_SECURITY_MANAGER = None CUSTOM_SECURITY_MANAGER = None
SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_TRACK_MODIFICATIONS = False

69
superset/migrations/versions/7fcdcde0761c_.py Normal file
View File

@ -0,0 +1,69 @@
"""Reduce position_json size by remove extra space and component id prefix
Revision ID: 7fcdcde0761c
Revises: c18bd4186f15
Create Date: 2018-08-01 11:47:02.233971
"""
# revision identifiers, used by Alembic.
import json
import re
from alembic import op
import sqlalchemy as sa
from sqlalchemy import (
Table, Column,
Integer, String, Text, ForeignKey,
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from superset import db
revision = '7fcdcde0761c'
down_revision = 'c18bd4186f15'
Base = declarative_base()
class Dashboard(Base):
"""Declarative class to do query in upgrade"""
__tablename__ = 'dashboards'
id = sa.Column(sa.Integer, primary_key=True)
dashboard_title = sa.Column(String(500))
position_json = sa.Column(sa.Text)
def is_v2_dash(positions):
return (
isinstance(positions, dict) and
positions.get('DASHBOARD_VERSION_KEY') == 'v2'
)
def upgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
dashboards = session.query(Dashboard).all()
for i, dashboard in enumerate(dashboards):
original_text = dashboard.position_json or ''
position_json = json.loads(original_text or '{}')
if is_v2_dash(position_json):
# re-dump the json data and remove leading and trailing white spaces
text = json.dumps(
position_json, indent=None, separators=(',', ':'), sort_keys=True)
# remove DASHBOARD_ and _TYPE prefix/suffix in all the component ids
text = re.sub(r'DASHBOARD_(?!VERSION)', '', text)
text = text.replace('_TYPE', '')
dashboard.position_json = text
print('dash id:{} position_json size from {} to {}'
.format(dashboard.id, len(original_text), len(text)))
session.merge(dashboard)
session.commit()
def downgrade():
pass

1
superset/views/base.py
View File

@ -26,6 +26,7 @@ from superset.translations.utils import get_language_pack
FRONTEND_CONF_KEYS = ( FRONTEND_CONF_KEYS = (
'SUPERSET_WEBSERVER_TIMEOUT', 'SUPERSET_WEBSERVER_TIMEOUT',
'SUPERSET_DASHBOARD_POSITION_DATA_LIMIT',
'ENABLE_JAVASCRIPT_CONTROLS', 'ENABLE_JAVASCRIPT_CONTROLS',
) )

3
superset/views/core.py
View File

@ -1643,6 +1643,9 @@ class Superset(BaseSupersetView):
session.merge(slc) session.merge(slc)
session.flush() session.flush()
# remove leading and trailing white spaces in the dumped json
dashboard.position_json = json.dumps(
positions, indent=None, separators=(',', ':'), sort_keys=True)
dashboard.position_json = json.dumps(positions, sort_keys=True) dashboard.position_json = json.dumps(positions, sort_keys=True)
md = dashboard.params_dict md = dashboard.params_dict
dashboard.css = data.get('css') dashboard.css = data.get('css')