mirror of https://github.com/apache/superset.git
Reduce dashboard position_json data size (#5543)
This commit is contained in:
parent
e1f4db8e24
commit
2e2c9806b2
|
@ -10,94 +10,91 @@ import {
|
|||
describe('findFirstParentContainer', () => {
|
||||
const mockGridLayout = {
|
||||
DASHBOARD_VERSION_KEY: 'v2',
|
||||
DASHBOARD_ROOT_ID: {
|
||||
type: 'DASHBOARD_ROOT_TYPE',
|
||||
id: 'DASHBOARD_ROOT_ID',
|
||||
children: ['DASHBOARD_GRID_ID'],
|
||||
ROOT_ID: {
|
||||
type: 'ROOT',
|
||||
id: 'ROOT_ID',
|
||||
children: ['GRID_ID'],
|
||||
},
|
||||
DASHBOARD_GRID_ID: {
|
||||
type: 'DASHBOARD_GRID_TYPE',
|
||||
id: 'DASHBOARD_GRID_ID',
|
||||
children: ['DASHBOARD_ROW_TYPE-Bk45URrlQ'],
|
||||
GRID_ID: {
|
||||
type: 'GRID',
|
||||
id: 'GRID_ID',
|
||||
children: ['ROW-Bk45URrlQ'],
|
||||
},
|
||||
'DASHBOARD_ROW_TYPE-Bk45URrlQ': {
|
||||
type: 'DASHBOARD_ROW_TYPE',
|
||||
id: 'DASHBOARD_ROW_TYPE-Bk45URrlQ',
|
||||
children: ['DASHBOARD_CHART_TYPE-ryxVc8RHlX'],
|
||||
'ROW-Bk45URrlQ': {
|
||||
type: 'ROW',
|
||||
id: 'ROW-Bk45URrlQ',
|
||||
children: ['CHART-ryxVc8RHlX'],
|
||||
},
|
||||
'DASHBOARD_CHART_TYPE-ryxVc8RHlX': {
|
||||
type: 'DASHBOARD_CHART_TYPE',
|
||||
id: 'DASHBOARD_CHART_TYPE-ryxVc8RHlX',
|
||||
'CHART-ryxVc8RHlX': {
|
||||
type: 'CHART',
|
||||
id: 'CHART-ryxVc8RHlX',
|
||||
children: [],
|
||||
},
|
||||
DASHBOARD_HEADER_ID: {
|
||||
id: 'DASHBOARD_HEADER_ID',
|
||||
type: 'DASHBOARD_HEADER_TYPE',
|
||||
HEADER_ID: {
|
||||
id: 'HEADER_ID',
|
||||
type: 'HEADER',
|
||||
},
|
||||
};
|
||||
const mockTabsLayout = {
|
||||
'DASHBOARD_CHART_TYPE-S1gilYABe7': {
|
||||
'CHART-S1gilYABe7': {
|
||||
children: [],
|
||||
id: 'DASHBOARD_CHART_TYPE-S1gilYABe7',
|
||||
type: 'DASHBOARD_CHART_TYPE',
|
||||
id: 'CHART-S1gilYABe7',
|
||||
type: 'CHART',
|
||||
},
|
||||
'DASHBOARD_CHART_TYPE-SJli5K0HlQ': {
|
||||
'CHART-SJli5K0HlQ': {
|
||||
children: [],
|
||||
id: 'DASHBOARD_CHART_TYPE-SJli5K0HlQ',
|
||||
type: 'DASHBOARD_CHART_TYPE',
|
||||
id: 'CHART-SJli5K0HlQ',
|
||||
type: 'CHART',
|
||||
},
|
||||
DASHBOARD_GRID_ID: {
|
||||
GRID_ID: {
|
||||
children: [],
|
||||
id: 'DASHBOARD_GRID_ID',
|
||||
type: 'DASHBOARD_GRID_TYPE',
|
||||
id: 'GRID_ID',
|
||||
type: 'GRID',
|
||||
},
|
||||
DASHBOARD_HEADER_ID: {
|
||||
id: 'DASHBOARD_HEADER_ID',
|
||||
type: 'DASHBOARD_HEADER_TYPE',
|
||||
HEADER_ID: {
|
||||
id: 'HEADER_ID',
|
||||
type: 'HEADER',
|
||||
},
|
||||
DASHBOARD_ROOT_ID: {
|
||||
children: ['DASHBOARD_TABS_TYPE-SkgJ5t0Bem'],
|
||||
id: 'DASHBOARD_ROOT_ID',
|
||||
type: 'DASHBOARD_ROOT_TYPE',
|
||||
ROOT_ID: {
|
||||
children: ['TABS-SkgJ5t0Bem'],
|
||||
id: 'ROOT_ID',
|
||||
type: 'ROOT',
|
||||
},
|
||||
'DASHBOARD_ROW_TYPE-S1B8-JLgX': {
|
||||
children: ['DASHBOARD_CHART_TYPE-SJli5K0HlQ'],
|
||||
id: 'DASHBOARD_ROW_TYPE-S1B8-JLgX',
|
||||
type: 'DASHBOARD_ROW_TYPE',
|
||||
'ROW-S1B8-JLgX': {
|
||||
children: ['CHART-SJli5K0HlQ'],
|
||||
id: 'ROW-S1B8-JLgX',
|
||||
type: 'ROW',
|
||||
},
|
||||
'DASHBOARD_ROW_TYPE-S1bUb1Ilm': {
|
||||
children: ['DASHBOARD_CHART_TYPE-S1gilYABe7'],
|
||||
id: 'DASHBOARD_ROW_TYPE-S1bUb1Ilm',
|
||||
type: 'DASHBOARD_ROW_TYPE',
|
||||
'ROW-S1bUb1Ilm': {
|
||||
children: ['CHART-S1gilYABe7'],
|
||||
id: 'ROW-S1bUb1Ilm',
|
||||
type: 'ROW',
|
||||
},
|
||||
'DASHBOARD_TABS_TYPE-ByeLSWyLe7': {
|
||||
children: ['DASHBOARD_TAB_TYPE-BJbLSZ1UeQ'],
|
||||
id: 'DASHBOARD_TABS_TYPE-ByeLSWyLe7',
|
||||
type: 'DASHBOARD_TABS_TYPE',
|
||||
'TABS-ByeLSWyLe7': {
|
||||
children: ['TAB-BJbLSZ1UeQ'],
|
||||
id: 'TABS-ByeLSWyLe7',
|
||||
type: 'TABS',
|
||||
},
|
||||
'DASHBOARD_TABS_TYPE-SkgJ5t0Bem': {
|
||||
children: [
|
||||
'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
|
||||
'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
|
||||
],
|
||||
id: 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem',
|
||||
'TABS-SkgJ5t0Bem': {
|
||||
children: ['TAB-HkWJcFCHxQ', 'TAB-ByDBbkLlQ'],
|
||||
id: 'TABS-SkgJ5t0Bem',
|
||||
meta: {},
|
||||
type: 'DASHBOARD_TABS_TYPE',
|
||||
type: 'TABS',
|
||||
},
|
||||
'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ': {
|
||||
children: ['DASHBOARD_ROW_TYPE-S1bUb1Ilm'],
|
||||
id: 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ',
|
||||
type: 'DASHBOARD_TAB_TYPE',
|
||||
'TAB-BJbLSZ1UeQ': {
|
||||
children: ['ROW-S1bUb1Ilm'],
|
||||
id: 'TAB-BJbLSZ1UeQ',
|
||||
type: 'TAB',
|
||||
},
|
||||
'DASHBOARD_TAB_TYPE-ByDBbkLlQ': {
|
||||
children: ['DASHBOARD_ROW_TYPE-S1B8-JLgX'],
|
||||
id: 'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
|
||||
type: 'DASHBOARD_TAB_TYPE',
|
||||
'TAB-ByDBbkLlQ': {
|
||||
children: ['ROW-S1B8-JLgX'],
|
||||
id: 'TAB-ByDBbkLlQ',
|
||||
type: 'TAB',
|
||||
},
|
||||
'DASHBOARD_TAB_TYPE-HkWJcFCHxQ': {
|
||||
children: ['DASHBOARD_TABS_TYPE-ByeLSWyLe7'],
|
||||
id: 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
|
||||
type: 'DASHBOARD_TAB_TYPE',
|
||||
'TAB-HkWJcFCHxQ': {
|
||||
children: ['TABS-ByeLSWyLe7'],
|
||||
id: 'TAB-HkWJcFCHxQ',
|
||||
type: 'TAB',
|
||||
},
|
||||
DASHBOARD_VERSION_KEY: 'v2',
|
||||
};
|
||||
|
|
|
@ -10,11 +10,16 @@ import UndoRedoKeylisteners from './UndoRedoKeylisteners';
|
|||
|
||||
import { chartPropShape } from '../util/propShapes';
|
||||
import { t } from '../../locales';
|
||||
import { UNDO_LIMIT, SAVE_TYPE_OVERWRITE } from '../util/constants';
|
||||
import {
|
||||
UNDO_LIMIT,
|
||||
SAVE_TYPE_OVERWRITE,
|
||||
DASHBOARD_POSITION_DATA_LIMIT,
|
||||
} from '../util/constants';
|
||||
|
||||
const propTypes = {
|
||||
addSuccessToast: PropTypes.func.isRequired,
|
||||
addDangerToast: PropTypes.func.isRequired,
|
||||
addWarningToast: PropTypes.func.isRequired,
|
||||
dashboardInfo: PropTypes.object.isRequired,
|
||||
dashboardTitle: PropTypes.string.isRequired,
|
||||
charts: PropTypes.objectOf(chartPropShape).isRequired,
|
||||
|
@ -143,7 +148,24 @@ class Header extends React.PureComponent {
|
|||
default_filters: JSON.stringify(filters),
|
||||
};
|
||||
|
||||
this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
|
||||
// make sure positions data less than DB storage limitation:
|
||||
const positionJSONLength = JSON.stringify(positions).length;
|
||||
const limit =
|
||||
dashboardInfo.common.conf.SUPERSET_DASHBOARD_POSITION_DATA_LIMIT ||
|
||||
DASHBOARD_POSITION_DATA_LIMIT;
|
||||
if (positionJSONLength >= limit) {
|
||||
this.props.addDangerToast(
|
||||
t(
|
||||
'Your dashboard is too large. Please reduce the size before save it.',
|
||||
),
|
||||
);
|
||||
} else {
|
||||
if (positionJSONLength >= limit * 0.9) {
|
||||
this.props.addWarningToast('Your dashboard is near the size limit.');
|
||||
}
|
||||
|
||||
this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
|
|
|
@ -23,7 +23,11 @@ import {
|
|||
updateDashboardTitle,
|
||||
} from '../actions/dashboardLayout';
|
||||
|
||||
import { addSuccessToast, addDangerToast } from '../../messageToasts/actions';
|
||||
import {
|
||||
addSuccessToast,
|
||||
addDangerToast,
|
||||
addWarningToast,
|
||||
} from '../../messageToasts/actions';
|
||||
|
||||
import { DASHBOARD_HEADER_ID } from '../util/constants';
|
||||
|
||||
|
@ -59,6 +63,7 @@ function mapDispatchToProps(dispatch) {
|
|||
{
|
||||
addSuccessToast,
|
||||
addDangerToast,
|
||||
addWarningToast,
|
||||
onUndo: undoLayoutAction,
|
||||
onRedo: redoLayoutAction,
|
||||
setEditMode,
|
||||
|
|
|
@ -1,15 +1,15 @@
|
|||
export const CHART_TYPE = 'DASHBOARD_CHART_TYPE';
|
||||
export const COLUMN_TYPE = 'DASHBOARD_COLUMN_TYPE';
|
||||
export const DASHBOARD_HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
|
||||
export const DASHBOARD_GRID_TYPE = 'DASHBOARD_GRID_TYPE';
|
||||
export const DASHBOARD_ROOT_TYPE = 'DASHBOARD_ROOT_TYPE';
|
||||
export const DIVIDER_TYPE = 'DASHBOARD_DIVIDER_TYPE';
|
||||
export const HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
|
||||
export const MARKDOWN_TYPE = 'DASHBOARD_MARKDOWN_TYPE';
|
||||
export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE_TYPE';
|
||||
export const ROW_TYPE = 'DASHBOARD_ROW_TYPE';
|
||||
export const TABS_TYPE = 'DASHBOARD_TABS_TYPE';
|
||||
export const TAB_TYPE = 'DASHBOARD_TAB_TYPE';
|
||||
export const CHART_TYPE = 'CHART';
|
||||
export const COLUMN_TYPE = 'COLUMN';
|
||||
export const DASHBOARD_HEADER_TYPE = 'HEADER';
|
||||
export const DASHBOARD_GRID_TYPE = 'GRID';
|
||||
export const DASHBOARD_ROOT_TYPE = 'ROOT';
|
||||
export const DIVIDER_TYPE = 'DIVIDER';
|
||||
export const HEADER_TYPE = 'HEADER';
|
||||
export const MARKDOWN_TYPE = 'MARKDOWN';
|
||||
export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE';
|
||||
export const ROW_TYPE = 'ROW';
|
||||
export const TABS_TYPE = 'TABS';
|
||||
export const TAB_TYPE = 'TAB';
|
||||
|
||||
export default {
|
||||
CHART_TYPE,
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
// Ids
|
||||
export const DASHBOARD_GRID_ID = 'DASHBOARD_GRID_ID';
|
||||
export const DASHBOARD_HEADER_ID = 'DASHBOARD_HEADER_ID';
|
||||
export const DASHBOARD_ROOT_ID = 'DASHBOARD_ROOT_ID';
|
||||
export const DASHBOARD_GRID_ID = 'GRID_ID';
|
||||
export const DASHBOARD_HEADER_ID = 'HEADER_ID';
|
||||
export const DASHBOARD_ROOT_ID = 'ROOT_ID';
|
||||
export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY';
|
||||
|
||||
export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID';
|
||||
|
@ -40,3 +40,7 @@ export const UNDO_LIMIT = 50;
|
|||
// save dash options
|
||||
export const SAVE_TYPE_OVERWRITE = 'overwrite';
|
||||
export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard';
|
||||
|
||||
// default dashboard layout data size limit
|
||||
// could be overwritten by server-side config
|
||||
export const DASHBOARD_POSITION_DATA_LIMIT = 65535;
|
||||
|
|
|
@ -49,6 +49,7 @@ SUPERSET_CELERY_WORKERS = 32 # deprecated
|
|||
SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
|
||||
SUPERSET_WEBSERVER_PORT = 8088
|
||||
SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated
|
||||
SUPERSET_DASHBOARD_POSITION_DATA_LIMIT = 65535
|
||||
EMAIL_NOTIFICATIONS = False
|
||||
CUSTOM_SECURITY_MANAGER = None
|
||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
|
|
|
@ -0,0 +1,69 @@
|
|||
"""Reduce position_json size by remove extra space and component id prefix
|
||||
|
||||
Revision ID: 7fcdcde0761c
|
||||
Revises: c18bd4186f15
|
||||
Create Date: 2018-08-01 11:47:02.233971
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
import json
|
||||
import re
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import (
|
||||
Table, Column,
|
||||
Integer, String, Text, ForeignKey,
|
||||
)
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from superset import db
|
||||
|
||||
revision = '7fcdcde0761c'
|
||||
down_revision = 'c18bd4186f15'
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class Dashboard(Base):
|
||||
"""Declarative class to do query in upgrade"""
|
||||
__tablename__ = 'dashboards'
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
dashboard_title = sa.Column(String(500))
|
||||
position_json = sa.Column(sa.Text)
|
||||
|
||||
|
||||
def is_v2_dash(positions):
|
||||
return (
|
||||
isinstance(positions, dict) and
|
||||
positions.get('DASHBOARD_VERSION_KEY') == 'v2'
|
||||
)
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = db.Session(bind=bind)
|
||||
|
||||
dashboards = session.query(Dashboard).all()
|
||||
for i, dashboard in enumerate(dashboards):
|
||||
original_text = dashboard.position_json or ''
|
||||
position_json = json.loads(original_text or '{}')
|
||||
if is_v2_dash(position_json):
|
||||
# re-dump the json data and remove leading and trailing white spaces
|
||||
text = json.dumps(
|
||||
position_json, indent=None, separators=(',', ':'), sort_keys=True)
|
||||
# remove DASHBOARD_ and _TYPE prefix/suffix in all the component ids
|
||||
text = re.sub(r'DASHBOARD_(?!VERSION)', '', text)
|
||||
text = text.replace('_TYPE', '')
|
||||
|
||||
dashboard.position_json = text
|
||||
print('dash id:{} position_json size from {} to {}'
|
||||
.format(dashboard.id, len(original_text), len(text)))
|
||||
session.merge(dashboard)
|
||||
session.commit()
|
||||
|
||||
|
||||
def downgrade():
|
||||
pass
|
|
@ -26,6 +26,7 @@ from superset.translations.utils import get_language_pack
|
|||
|
||||
FRONTEND_CONF_KEYS = (
|
||||
'SUPERSET_WEBSERVER_TIMEOUT',
|
||||
'SUPERSET_DASHBOARD_POSITION_DATA_LIMIT',
|
||||
'ENABLE_JAVASCRIPT_CONTROLS',
|
||||
)
|
||||
|
||||
|
|
|
@ -1643,6 +1643,9 @@ class Superset(BaseSupersetView):
|
|||
session.merge(slc)
|
||||
session.flush()
|
||||
|
||||
# remove leading and trailing white spaces in the dumped json
|
||||
dashboard.position_json = json.dumps(
|
||||
positions, indent=None, separators=(',', ':'), sort_keys=True)
|
||||
dashboard.position_json = json.dumps(positions, sort_keys=True)
|
||||
md = dashboard.params_dict
|
||||
dashboard.css = data.get('css')
|
||||
|
|
Loading…
Reference in New Issue