From d522292b01baf82b9952d5785740dfa8c88e4f53 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 9 Mar 2018 14:54:39 -0800 Subject: [PATCH] [sql lab] option to disable cross schema search (#4551) * [sql lab] disable cross schema search This is killing our metastore as people type it emits large all-table-dump as they hit the keystroke. It never returns as it times out and hammers the poor metastore. Also some improvements around the disabling the table select on the left panel and having the table name not be sticky. * typo --- .../SqlLab/components/SqlEditor.jsx | 1 + .../SqlLab/components/SqlEditorLeftBar.jsx | 14 +++++++-- .../sqllab/SqlEditorLeftBar_spec.jsx | 1 + .../sqllab/TabbedSqlEditors_spec.jsx | 1 + ...3c581_allow_multi_schema_metadata_fetch.py | 30 +++++++++++++++++++ superset/models/core.py | 7 +++++ superset/views/core.py | 9 +++++- 7 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 superset/migrations/versions/e68c4473c581_allow_multi_schema_metadata_fetch.py diff --git a/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx b/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx index 3eab16c3b9..682c7053aa 100644 --- a/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx @@ -241,6 +241,7 @@ class SqlEditor extends React.PureComponent { >
@@ -186,7 +195,6 @@ class SqlEditorLeftBar extends React.PureComponent { name="select-table" ref="selectTable" isLoading={this.state.tableLoading} - value={this.state.tableName} placeholder={t('Add a table (%s)', this.state.tableOptions.length)} autosize={false} onChange={this.changeTable.bind(this)} @@ -199,8 +207,8 @@ class SqlEditorLeftBar extends React.PureComponent { async name="async-select-table" ref="selectTable" - value={this.state.tableName} - placeholder={t('Type to search ...')} + placeholder={tableSelectPlaceholder} + disabled={tableSelectDisabled} autosize={false} onChange={this.changeTable.bind(this)} loadOptions={this.getTableNamesBySubStr.bind(this)} diff --git a/superset/assets/spec/javascripts/sqllab/SqlEditorLeftBar_spec.jsx b/superset/assets/spec/javascripts/sqllab/SqlEditorLeftBar_spec.jsx index 9c90661fee..5755c658d9 100644 --- a/superset/assets/spec/javascripts/sqllab/SqlEditorLeftBar_spec.jsx +++ b/superset/assets/spec/javascripts/sqllab/SqlEditorLeftBar_spec.jsx @@ -23,6 +23,7 @@ describe('SqlEditorLeftBar', () => { }, tables: [table], queryEditor: defaultQueryEditor, + database: {}, height: 0, }; diff --git a/superset/assets/spec/javascripts/sqllab/TabbedSqlEditors_spec.jsx b/superset/assets/spec/javascripts/sqllab/TabbedSqlEditors_spec.jsx index 35f8a45ee7..11e889eefb 100644 --- a/superset/assets/spec/javascripts/sqllab/TabbedSqlEditors_spec.jsx +++ b/superset/assets/spec/javascripts/sqllab/TabbedSqlEditors_spec.jsx @@ -51,6 +51,7 @@ describe('TabbedSqlEditors', () => { tabHistory: initialState.tabHistory, editorHeight: '', getHeight: () => ('100px'), + database: {}, }; const getWrapper = () => ( shallow(, { diff --git a/superset/migrations/versions/e68c4473c581_allow_multi_schema_metadata_fetch.py b/superset/migrations/versions/e68c4473c581_allow_multi_schema_metadata_fetch.py new file mode 100644 index 0000000000..dab7cb6079 --- /dev/null +++ b/superset/migrations/versions/e68c4473c581_allow_multi_schema_metadata_fetch.py @@ -0,0 +1,30 @@ +"""allow_multi_schema_metadata_fetch + +Revision ID: e68c4473c581 +Revises: e866bd2d4976 +Create Date: 2018-03-06 12:24:30.896293 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e68c4473c581' +down_revision = 'e866bd2d4976' + + +def upgrade(): + + op.add_column( + 'dbs', + sa.Column( + 'allow_multi_schema_metadata_fetch', + sa.Boolean(), + nullable=True, + default=True, + ), + ) + + +def downgrade(): + op.drop_column('dbs', 'allow_multi_schema_metadata_fetch') diff --git a/superset/models/core.py b/superset/models/core.py index 9caa7adce4..b477f02e34 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -567,6 +567,7 @@ class Database(Model, AuditMixinNullable, ImportMixin): allow_ctas = Column(Boolean, default=False) allow_dml = Column(Boolean, default=False) force_ctas_schema = Column(String(250)) + allow_multi_schema_metadata_fetch = Column(Boolean, default=True) extra = Column(Text, default=textwrap.dedent("""\ { "metadata_params": {}, @@ -593,6 +594,8 @@ class Database(Model, AuditMixinNullable, ImportMixin): return { 'name': self.database_name, 'backend': self.backend, + 'allow_multi_schema_metadata_fetch': + self.allow_multi_schema_metadata_fetch, } @property @@ -736,6 +739,8 @@ class Database(Model, AuditMixinNullable, ImportMixin): def all_table_names(self, schema=None, force=False): if not schema: + if not self.allow_multi_schema_metadata_fetch: + return [] tables_dict = self.db_engine_spec.fetch_result_sets( self, 'table', force=force) return tables_dict.get('', []) @@ -744,6 +749,8 @@ class Database(Model, AuditMixinNullable, ImportMixin): def all_view_names(self, schema=None, force=False): if not schema: + if not self.allow_multi_schema_metadata_fetch: + return [] views_dict = self.db_engine_spec.fetch_result_sets( self, 'view', force=force) return views_dict.get('', []) diff --git a/superset/views/core.py b/superset/views/core.py index 4330e7d7ee..3b18e82d20 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -199,7 +199,9 @@ class DatabaseView(SupersetModelView, DeleteMixin, YamlExportMixin): # noqa add_columns = [ 'database_name', 'sqlalchemy_uri', 'cache_timeout', 'extra', 'expose_in_sqllab', 'allow_run_sync', 'allow_run_async', - 'allow_ctas', 'allow_dml', 'force_ctas_schema', 'impersonate_user'] + 'allow_ctas', 'allow_dml', 'force_ctas_schema', 'impersonate_user', + 'allow_multi_schema_metadata_fetch', + ] search_exclude_columns = ( 'password', 'tables', 'created_by', 'changed_by', 'queries', 'saved_queries') @@ -258,6 +260,10 @@ class DatabaseView(SupersetModelView, DeleteMixin, YamlExportMixin): # noqa 'If Hive and hive.server2.enable.doAs is enabled, will run the queries as ' 'service account, but impersonate the currently logged on user ' 'via hive.server2.proxy.user property.'), + 'allow_multi_schema_metadata_fetch': _( + 'Allow SQL Lab to fetch a list of all tables and all views across ' + 'all database schemas. For large data warehouse with thousands of ' + 'tables, this can be expensive and put strain on the system.'), } label_columns = { 'expose_in_sqllab': _('Expose in SQL Lab'), @@ -314,6 +320,7 @@ class DatabaseAsync(DatabaseView): 'id', 'database_name', 'expose_in_sqllab', 'allow_ctas', 'force_ctas_schema', 'allow_run_async', 'allow_run_sync', 'allow_dml', + 'allow_multi_schema_metadata_fetch', ]