pt/superset
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2024-09-12 16:49:40 -04:00
Code Issues Packages Projects Releases Wiki Activity

perf: memoize db_engine_spec in database (#14638)

Browse Source 
* perf: memoize db_engine_spec in sqla table classes

* remove extended cypress timeouts
This commit is contained in:
Ville Brofeldt 2021-05-14 12:49:35 +03:00 committed by GitHub
parent bf90885828
commit 97c9e37c24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 36 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
superset-frontend/cypress-base/cypress/integration/dashboard/dashboard.helper.ts
View File

@ -45,9 +45,7 @@ export interface ChartSpec {
export function getChartGridComponent({ name, viz }: ChartSpec) { export function getChartGridComponent({ name, viz }: ChartSpec) {
return cy return cy
.get(`[data-test="chart-grid-component"][data-test-chart-name="${name}"]`, { .get(`[data-test="chart-grid-component"][data-test-chart-name="${name}"]`)
timeout: 30000,
})
.should('have.attr', 'data-test-viz-type', viz); .should('have.attr', 'data-test-viz-type', viz);
} }

4
superset-frontend/cypress-base/cypress/integration/dashboard/load.test.ts
View File

@ -44,9 +44,7 @@ describe('Dashboard load', () => {
it('should load in edit/standalone mode', () => { it('should load in edit/standalone mode', () => {
cy.visit(`${WORLD_HEALTH_DASHBOARD}?edit=true&standalone=true`); cy.visit(`${WORLD_HEALTH_DASHBOARD}?edit=true&standalone=true`);
cy.get('[data-test="discard-changes-button"]', { timeout: 10000 }).should( cy.get('[data-test="discard-changes-button"]').should('be.visible');
'be.visible',
);
cy.get('#app-menu').should('not.exist'); cy.get('#app-menu').should('not.exist');
}); });

2
superset-frontend/src/datasource/ChangeDatasourceModal.tsx
View File

@ -163,7 +163,7 @@ const ChangeDatasourceModal: FunctionComponent<ChangeDatasourceModalProps> = ({
const handleChangeConfirm = () => { const handleChangeConfirm = () => {
SupersetClient.get({ SupersetClient.get({
endpoint: `/datasource/get/${confirmedDataset?.type}/${confirmedDataset?.id}`, endpoint: `/datasource/get/${confirmedDataset?.type}/${confirmedDataset?.id}/`,
}) })
.then(({ json }) => { .then(({ json }) => {
onDatasourceSave(json); onDatasourceSave(json);

64
superset/connectors/sqla/models.py
View File

@ -22,7 +22,7 @@ from collections import defaultdict, OrderedDict
from contextlib import closing from contextlib import closing
from dataclasses import dataclass, field # pylint: disable=wrong-import-order from dataclasses import dataclass, field # pylint: disable=wrong-import-order
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Any, Dict, Hashable, List, NamedTuple, Optional, Tuple, Union from typing import Any, Dict, Hashable, List, NamedTuple, Optional, Tuple, Type, Union
import pandas as pd import pandas as pd
import sqlalchemy as sa import sqlalchemy as sa
@ -57,7 +57,7 @@ from sqlalchemy.types import TypeEngine
from superset import app, db, is_feature_enabled, security_manager from superset import app, db, is_feature_enabled, security_manager
from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
from superset.db_engine_specs.base import TimestampExpression from superset.db_engine_specs.base import BaseEngineSpec, TimestampExpression
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import ( from superset.exceptions import (
QueryObjectValidationError, QueryObjectValidationError,
@ -196,30 +196,21 @@ class TableColumn(Model, BaseColumn):
""" """
Check if the column has a boolean datatype. Check if the column has a boolean datatype.
""" """
column_spec = self.table.database.db_engine_spec.get_column_spec(self.type) return self.type_generic == GenericDataType.BOOLEAN
if column_spec is None:
return False
return column_spec.generic_type == GenericDataType.BOOLEAN
@property @property
def is_numeric(self) -> bool: def is_numeric(self) -> bool:
""" """
Check if the column has a numeric datatype. Check if the column has a numeric datatype.
""" """
column_spec = self.table.database.db_engine_spec.get_column_spec(self.type) return self.type_generic == GenericDataType.NUMERIC
if column_spec is None:
return False
return column_spec.generic_type == GenericDataType.NUMERIC
@property @property
def is_string(self) -> bool: def is_string(self) -> bool:
""" """
Check if the column has a string datatype. Check if the column has a string datatype.
""" """
column_spec = self.table.database.db_engine_spec.get_column_spec(self.type) return self.type_generic == GenericDataType.STRING
if column_spec is None:
return False
return column_spec.generic_type == GenericDataType.STRING
@property @property
def is_temporal(self) -> bool: def is_temporal(self) -> bool:
@ -231,14 +222,20 @@ class TableColumn(Model, BaseColumn):
""" """
if self.is_dttm is not None: if self.is_dttm is not None:
return self.is_dttm return self.is_dttm
column_spec = self.table.database.db_engine_spec.get_column_spec(self.type) return self.type_generic == GenericDataType.TEMPORAL
if column_spec is None:
return False @property
return column_spec.is_dttm def db_engine_spec(self) -> Type[BaseEngineSpec]:
return self.table.db_engine_spec
@property
def type_generic(self) -> Optional[utils.GenericDataType]:
column_spec = self.db_engine_spec.get_column_spec(self.type)
return column_spec.generic_type if column_spec else None
def get_sqla_col(self, label: Optional[str] = None) -> Column: def get_sqla_col(self, label: Optional[str] = None) -> Column:
label = label or self.column_name label = label or self.column_name
db_engine_spec = self.table.database.db_engine_spec db_engine_spec = self.db_engine_spec
column_spec = db_engine_spec.get_column_spec(self.type) column_spec = db_engine_spec.get_column_spec(self.type)
type_ = column_spec.sqla_type if column_spec else None type_ = column_spec.sqla_type if column_spec else None
if self.expression: if self.expression:
@ -290,7 +287,6 @@ class TableColumn(Model, BaseColumn):
""" """
label = label or utils.DTTM_ALIAS label = label or utils.DTTM_ALIAS
db_ = self.table.database
pdf = self.python_date_format pdf = self.python_date_format
is_epoch = pdf in ("epoch_s", "epoch_ms") is_epoch = pdf in ("epoch_s", "epoch_ms")
if not self.expression and not time_grain and not is_epoch: if not self.expression and not time_grain and not is_epoch:
@ -300,7 +296,7 @@ class TableColumn(Model, BaseColumn):
col = literal_column(self.expression) col = literal_column(self.expression)
else: else:
col = column(self.column_name) col = column(self.column_name)
time_expr = db_.db_engine_spec.get_timestamp_expr( time_expr = self.db_engine_spec.get_timestamp_expr(
col, pdf, time_grain, self.type col, pdf, time_grain, self.type
) )
return self.table.make_sqla_column_compatible(time_expr, label) return self.table.make_sqla_column_compatible(time_expr, label)
@ -313,11 +309,7 @@ class TableColumn(Model, BaseColumn):
], ],
) -> str: ) -> str:
"""Convert datetime object to a SQL expression string""" """Convert datetime object to a SQL expression string"""
sql = ( sql = self.db_engine_spec.convert_dttm(self.type, dttm) if self.type else None
self.table.database.db_engine_spec.convert_dttm(self.type, dttm)
if self.type
else None
)
if sql: if sql:
return sql return sql
@ -527,6 +519,10 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
def __repr__(self) -> str: def __repr__(self) -> str:
return self.name return self.name
@property
def db_engine_spec(self) -> Type[BaseEngineSpec]:
return self.database.db_engine_spec
@property @property
def changed_by_name(self) -> str: def changed_by_name(self) -> str:
if not self.changed_by: if not self.changed_by:
@ -636,7 +632,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
return self.database.sql_url + "?table_name=" + str(self.table_name) return self.database.sql_url + "?table_name=" + str(self.table_name)
def external_metadata(self) -> List[Dict[str, str]]: def external_metadata(self) -> List[Dict[str, str]]:
db_engine_spec = self.database.db_engine_spec db_engine_spec = self.db_engine_spec
if self.sql: if self.sql:
engine = self.database.get_sqla_engine(schema=self.schema) engine = self.database.get_sqla_engine(schema=self.schema)
sql = self.get_template_processor().process_template(self.sql) sql = self.get_template_processor().process_template(self.sql)
@ -815,9 +811,9 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
from_sql = self.get_rendered_sql(template_processor) from_sql = self.get_rendered_sql(template_processor)
parsed_query = ParsedQuery(from_sql) parsed_query = ParsedQuery(from_sql)
db_engine_spec = self.database.db_engine_spec
if not ( if not (
parsed_query.is_unknown() or db_engine_spec.is_readonly_query(parsed_query) parsed_query.is_unknown()
or self.db_engine_spec.is_readonly_query(parsed_query)
): ):
raise QueryObjectValidationError( raise QueryObjectValidationError(
_("Virtual dataset query must be read-only") _("Virtual dataset query must be read-only")
@ -889,7 +885,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
:return: either a sql alchemy column or label instance if supported by engine :return: either a sql alchemy column or label instance if supported by engine
""" """
label_expected = label or sqla_col.name label_expected = label or sqla_col.name
db_engine_spec = self.database.db_engine_spec db_engine_spec = self.db_engine_spec
# add quotes to tables # add quotes to tables
if db_engine_spec.allows_alias_in_select: if db_engine_spec.allows_alias_in_select:
label = db_engine_spec.make_label_compatible(label_expected) label = db_engine_spec.make_label_compatible(label_expected)
@ -909,7 +905,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
the same as a source column. In this case, we update the SELECT alias to the same as a source column. In this case, we update the SELECT alias to
another name to avoid the conflict. another name to avoid the conflict.
""" """
if self.database.db_engine_spec.allows_alias_to_source_column: if self.db_engine_spec.allows_alias_to_source_column:
return return
def is_alias_used_in_orderby(col: ColumnElement) -> bool: def is_alias_used_in_orderby(col: ColumnElement) -> bool:
@ -990,7 +986,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
extra_cache_keys: List[Any] = [] extra_cache_keys: List[Any] = []
template_kwargs["extra_cache_keys"] = extra_cache_keys template_kwargs["extra_cache_keys"] = extra_cache_keys
template_processor = self.get_template_processor(**template_kwargs) template_processor = self.get_template_processor(**template_kwargs)
db_engine_spec = self.database.db_engine_spec db_engine_spec = self.db_engine_spec
prequeries: List[str] = [] prequeries: List[str] = []
orderby = orderby or [] orderby = orderby or []
extras = extras or {} extras = extras or {}
@ -1469,7 +1465,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
logger.warning( logger.warning(
"Query %s on schema %s failed", sql, self.schema, exc_info=True "Query %s on schema %s failed", sql, self.schema, exc_info=True
) )
db_engine_spec = self.database.db_engine_spec db_engine_spec = self.db_engine_spec
errors = [ errors = [
dataclasses.asdict(error) for error in db_engine_spec.extract_errors(ex) dataclasses.asdict(error) for error in db_engine_spec.extract_errors(ex)
] ]
@ -1497,7 +1493,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
new_columns = self.external_metadata() new_columns = self.external_metadata()
metrics = [] metrics = []
any_date_col = None any_date_col = None
db_engine_spec = self.database.db_engine_spec db_engine_spec = self.db_engine_spec
old_columns = db.session.query(TableColumn).filter(TableColumn.table == self) old_columns = db.session.query(TableColumn).filter(TableColumn.table == self)
old_columns_by_name: Dict[str, TableColumn] = { old_columns_by_name: Dict[str, TableColumn] = {

1
superset/db_engine_specs/base.py
View File

@ -1255,6 +1255,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
) )
@classmethod @classmethod
@utils.memoized
def get_column_spec( def get_column_spec(
cls, cls,
native_type: Optional[str], native_type: Optional[str],

4
superset/models/core.py
View File

@ -568,10 +568,10 @@ class Database(
@property @property
def db_engine_spec(self) -> Type[db_engine_specs.BaseEngineSpec]: def db_engine_spec(self) -> Type[db_engine_specs.BaseEngineSpec]:
engines = db_engine_specs.get_engine_specs() return self.get_db_engine_spec_for_backend(self.backend)
return engines.get(self.backend, db_engine_specs.BaseEngineSpec)
@classmethod @classmethod
@utils.memoized
def get_db_engine_spec_for_backend( def get_db_engine_spec_for_backend(
cls, backend: str cls, backend: str
) -> Type[db_engine_specs.BaseEngineSpec]: ) -> Type[db_engine_specs.BaseEngineSpec]: