2019-01-15 18:53:27 -05:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
# distributed with this work for additional information
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
# under the License.
|
2018-03-15 20:53:34 -04:00
|
|
|
import textwrap
|
2019-06-24 01:37:41 -04:00
|
|
|
import unittest
|
2017-02-27 01:54:48 -05:00
|
|
|
|
2019-02-04 15:34:24 -05:00
|
|
|
import pandas
|
2017-02-27 01:54:48 -05:00
|
|
|
from sqlalchemy.engine.url import make_url
|
|
|
|
|
2018-07-26 18:20:23 -04:00
|
|
|
from superset import app, db
|
2017-03-10 12:11:51 -05:00
|
|
|
from superset.models.core import Database
|
2019-02-04 15:34:24 -05:00
|
|
|
from superset.utils.core import get_main_database, QueryStatus
|
2018-04-15 19:21:33 -04:00
|
|
|
from .base_tests import SupersetTestCase
|
2017-02-27 01:54:48 -05:00
|
|
|
|
|
|
|
|
2018-03-15 20:53:34 -04:00
|
|
|
class DatabaseModelTestCase(SupersetTestCase):
|
2017-04-10 18:36:58 -04:00
|
|
|
|
|
|
|
def test_database_schema_presto(self):
|
2017-02-27 01:54:48 -05:00
|
|
|
sqlalchemy_uri = 'presto://presto.airbnb.io:8080/hive/default'
|
|
|
|
model = Database(sqlalchemy_uri=sqlalchemy_uri)
|
|
|
|
|
2017-04-10 18:36:58 -04:00
|
|
|
db = make_url(model.get_sqla_engine().url).database
|
|
|
|
self.assertEquals('hive/default', db)
|
|
|
|
|
|
|
|
db = make_url(model.get_sqla_engine(schema='core_db').url).database
|
|
|
|
self.assertEquals('hive/core_db', db)
|
|
|
|
|
|
|
|
sqlalchemy_uri = 'presto://presto.airbnb.io:8080/hive'
|
2017-02-27 01:54:48 -05:00
|
|
|
model = Database(sqlalchemy_uri=sqlalchemy_uri)
|
|
|
|
|
2017-04-10 18:36:58 -04:00
|
|
|
db = make_url(model.get_sqla_engine().url).database
|
|
|
|
self.assertEquals('hive', db)
|
|
|
|
|
|
|
|
db = make_url(model.get_sqla_engine(schema='core_db').url).database
|
|
|
|
self.assertEquals('hive/core_db', db)
|
|
|
|
|
|
|
|
def test_database_schema_postgres(self):
|
|
|
|
sqlalchemy_uri = 'postgresql+psycopg2://postgres.airbnb.io:5439/prod'
|
2017-02-27 01:54:48 -05:00
|
|
|
model = Database(sqlalchemy_uri=sqlalchemy_uri)
|
|
|
|
|
2017-04-10 18:36:58 -04:00
|
|
|
db = make_url(model.get_sqla_engine().url).database
|
|
|
|
self.assertEquals('prod', db)
|
|
|
|
|
|
|
|
db = make_url(model.get_sqla_engine(schema='foo').url).database
|
|
|
|
self.assertEquals('prod', db)
|
|
|
|
|
2019-06-24 01:37:41 -04:00
|
|
|
@unittest.skipUnless(
|
|
|
|
SupersetTestCase.is_module_installed('thrift'), 'thrift not installed')
|
|
|
|
@unittest.skipUnless(
|
|
|
|
SupersetTestCase.is_module_installed('pyhive'), 'pyhive not installed')
|
2017-04-10 18:36:58 -04:00
|
|
|
def test_database_schema_hive(self):
|
2017-04-18 15:29:13 -04:00
|
|
|
sqlalchemy_uri = 'hive://hive@hive.airbnb.io:10000/default?auth=NOSASL'
|
2017-02-27 01:54:48 -05:00
|
|
|
model = Database(sqlalchemy_uri=sqlalchemy_uri)
|
2017-04-10 18:36:58 -04:00
|
|
|
db = make_url(model.get_sqla_engine().url).database
|
2017-04-18 15:29:13 -04:00
|
|
|
self.assertEquals('default', db)
|
2017-04-10 18:36:58 -04:00
|
|
|
|
|
|
|
db = make_url(model.get_sqla_engine(schema='core_db').url).database
|
2017-04-18 15:29:13 -04:00
|
|
|
self.assertEquals('core_db', db)
|
2017-02-27 01:54:48 -05:00
|
|
|
|
2019-06-24 01:37:41 -04:00
|
|
|
@unittest.skipUnless(
|
|
|
|
SupersetTestCase.is_module_installed('MySQLdb'), 'mysqlclient not installed')
|
2017-04-10 18:36:58 -04:00
|
|
|
def test_database_schema_mysql(self):
|
|
|
|
sqlalchemy_uri = 'mysql://root@localhost/superset'
|
2017-02-27 01:54:48 -05:00
|
|
|
model = Database(sqlalchemy_uri=sqlalchemy_uri)
|
2017-04-10 18:36:58 -04:00
|
|
|
|
|
|
|
db = make_url(model.get_sqla_engine().url).database
|
|
|
|
self.assertEquals('superset', db)
|
|
|
|
|
|
|
|
db = make_url(model.get_sqla_engine(schema='staging').url).database
|
|
|
|
self.assertEquals('staging', db)
|
2017-09-18 12:52:29 -04:00
|
|
|
|
2019-06-24 01:37:41 -04:00
|
|
|
@unittest.skipUnless(
|
|
|
|
SupersetTestCase.is_module_installed('MySQLdb'), 'mysqlclient not installed')
|
2017-09-18 12:52:29 -04:00
|
|
|
def test_database_impersonate_user(self):
|
|
|
|
uri = 'mysql://root@localhost'
|
|
|
|
example_user = 'giuseppe'
|
|
|
|
model = Database(sqlalchemy_uri=uri)
|
|
|
|
|
|
|
|
model.impersonate_user = True
|
|
|
|
user_name = make_url(model.get_sqla_engine(user_name=example_user).url).username
|
|
|
|
self.assertEquals(example_user, user_name)
|
|
|
|
|
|
|
|
model.impersonate_user = False
|
|
|
|
user_name = make_url(model.get_sqla_engine(user_name=example_user).url).username
|
|
|
|
self.assertNotEquals(example_user, user_name)
|
2018-03-15 20:53:34 -04:00
|
|
|
|
|
|
|
def test_select_star(self):
|
2018-09-06 17:55:48 -04:00
|
|
|
main_db = get_main_database(db.session)
|
2018-11-19 18:27:25 -05:00
|
|
|
table_name = 'energy_usage'
|
2018-03-15 20:53:34 -04:00
|
|
|
sql = main_db.select_star(
|
|
|
|
table_name, show_cols=False, latest_partition=False)
|
2018-12-02 16:50:49 -05:00
|
|
|
expected = textwrap.dedent(f"""\
|
2018-03-15 20:53:34 -04:00
|
|
|
SELECT *
|
|
|
|
FROM {table_name}
|
2018-12-02 16:50:49 -05:00
|
|
|
LIMIT 100""")
|
2018-03-15 20:53:34 -04:00
|
|
|
assert sql.startswith(expected)
|
|
|
|
|
|
|
|
sql = main_db.select_star(
|
|
|
|
table_name, show_cols=True, latest_partition=False)
|
2018-12-02 16:50:49 -05:00
|
|
|
expected = textwrap.dedent(f"""\
|
2018-11-19 18:27:25 -05:00
|
|
|
SELECT source,
|
|
|
|
target,
|
|
|
|
value
|
|
|
|
FROM energy_usage
|
2018-12-02 16:50:49 -05:00
|
|
|
LIMIT 100""")
|
2018-03-15 20:53:34 -04:00
|
|
|
assert sql.startswith(expected)
|
2018-04-18 19:17:28 -04:00
|
|
|
|
2018-05-29 17:20:17 -04:00
|
|
|
def test_single_statement(self):
|
2018-09-06 17:55:48 -04:00
|
|
|
main_db = get_main_database(db.session)
|
2018-05-29 17:20:17 -04:00
|
|
|
|
|
|
|
if main_db.backend == 'mysql':
|
|
|
|
df = main_db.get_df('SELECT 1', None)
|
|
|
|
self.assertEquals(df.iat[0, 0], 1)
|
|
|
|
|
|
|
|
df = main_db.get_df('SELECT 1;', None)
|
|
|
|
self.assertEquals(df.iat[0, 0], 1)
|
|
|
|
|
|
|
|
def test_multi_statement(self):
|
2018-09-06 17:55:48 -04:00
|
|
|
main_db = get_main_database(db.session)
|
2018-05-29 17:20:17 -04:00
|
|
|
|
|
|
|
if main_db.backend == 'mysql':
|
|
|
|
df = main_db.get_df('USE superset; SELECT 1', None)
|
|
|
|
self.assertEquals(df.iat[0, 0], 1)
|
|
|
|
|
|
|
|
df = main_db.get_df("USE superset; SELECT ';';", None)
|
|
|
|
self.assertEquals(df.iat[0, 0], ';')
|
|
|
|
|
2018-04-27 00:13:52 -04:00
|
|
|
|
|
|
|
class SqlaTableModelTestCase(SupersetTestCase):
|
|
|
|
|
|
|
|
def test_get_timestamp_expression(self):
|
|
|
|
tbl = self.get_table_by_name('birth_names')
|
|
|
|
ds_col = tbl.get_column('ds')
|
|
|
|
sqla_literal = ds_col.get_timestamp_expression(None)
|
|
|
|
self.assertEquals(str(sqla_literal.compile()), 'ds')
|
|
|
|
|
|
|
|
sqla_literal = ds_col.get_timestamp_expression('P1D')
|
|
|
|
compiled = '{}'.format(sqla_literal.compile())
|
|
|
|
if tbl.database.backend == 'mysql':
|
|
|
|
self.assertEquals(compiled, 'DATE(ds)')
|
|
|
|
|
2019-02-04 15:34:24 -05:00
|
|
|
prev_ds_expr = ds_col.expression
|
2018-04-27 00:13:52 -04:00
|
|
|
ds_col.expression = 'DATE_ADD(ds, 1)'
|
|
|
|
sqla_literal = ds_col.get_timestamp_expression('P1D')
|
|
|
|
compiled = '{}'.format(sqla_literal.compile())
|
|
|
|
if tbl.database.backend == 'mysql':
|
|
|
|
self.assertEquals(compiled, 'DATE(DATE_ADD(ds, 1))')
|
2019-02-04 15:34:24 -05:00
|
|
|
ds_col.expression = prev_ds_expr
|
2018-04-27 00:13:52 -04:00
|
|
|
|
|
|
|
def test_get_timestamp_expression_epoch(self):
|
|
|
|
tbl = self.get_table_by_name('birth_names')
|
|
|
|
ds_col = tbl.get_column('ds')
|
|
|
|
|
|
|
|
ds_col.expression = None
|
|
|
|
ds_col.python_date_format = 'epoch_s'
|
|
|
|
sqla_literal = ds_col.get_timestamp_expression(None)
|
|
|
|
compiled = '{}'.format(sqla_literal.compile())
|
|
|
|
if tbl.database.backend == 'mysql':
|
|
|
|
self.assertEquals(compiled, 'from_unixtime(ds)')
|
|
|
|
|
|
|
|
ds_col.python_date_format = 'epoch_s'
|
|
|
|
sqla_literal = ds_col.get_timestamp_expression('P1D')
|
|
|
|
compiled = '{}'.format(sqla_literal.compile())
|
|
|
|
if tbl.database.backend == 'mysql':
|
|
|
|
self.assertEquals(compiled, 'DATE(from_unixtime(ds))')
|
|
|
|
|
2019-02-04 15:34:24 -05:00
|
|
|
prev_ds_expr = ds_col.expression
|
2018-04-27 00:13:52 -04:00
|
|
|
ds_col.expression = 'DATE_ADD(ds, 1)'
|
|
|
|
sqla_literal = ds_col.get_timestamp_expression('P1D')
|
|
|
|
compiled = '{}'.format(sqla_literal.compile())
|
|
|
|
if tbl.database.backend == 'mysql':
|
|
|
|
self.assertEquals(compiled, 'DATE(from_unixtime(DATE_ADD(ds, 1)))')
|
2019-02-04 15:34:24 -05:00
|
|
|
ds_col.expression = prev_ds_expr
|
2018-04-27 00:13:52 -04:00
|
|
|
|
2019-02-04 15:34:24 -05:00
|
|
|
def query_with_expr_helper(self, is_timeseries, inner_join=True):
|
|
|
|
tbl = self.get_table_by_name('birth_names')
|
|
|
|
ds_col = tbl.get_column('ds')
|
|
|
|
ds_col.expression = None
|
|
|
|
ds_col.python_date_format = None
|
|
|
|
spec = self.get_database_by_id(tbl.database_id).db_engine_spec
|
|
|
|
if not spec.inner_joins and inner_join:
|
|
|
|
# if the db does not support inner joins, we cannot force it so
|
|
|
|
return None
|
|
|
|
old_inner_join = spec.inner_joins
|
|
|
|
spec.inner_joins = inner_join
|
|
|
|
arbitrary_gby = "state || gender || '_test'"
|
|
|
|
arbitrary_metric = (dict(label='arbitrary', expressionType='SQL',
|
|
|
|
sqlExpression='COUNT(1)'))
|
|
|
|
query_obj = dict(
|
|
|
|
groupby=[arbitrary_gby, 'name'],
|
|
|
|
metrics=[arbitrary_metric],
|
|
|
|
filter=[],
|
|
|
|
is_timeseries=is_timeseries,
|
|
|
|
prequeries=[],
|
|
|
|
columns=[],
|
|
|
|
granularity='ds',
|
|
|
|
from_dttm=None,
|
|
|
|
to_dttm=None,
|
|
|
|
is_prequery=False,
|
|
|
|
extras=dict(time_grain_sqla='P1Y'),
|
|
|
|
)
|
|
|
|
qr = tbl.query(query_obj)
|
|
|
|
self.assertEqual(qr.status, QueryStatus.SUCCESS)
|
|
|
|
sql = qr.query
|
|
|
|
self.assertIn(arbitrary_gby, sql)
|
|
|
|
self.assertIn('name', sql)
|
|
|
|
if inner_join and is_timeseries:
|
|
|
|
self.assertIn('JOIN', sql.upper())
|
|
|
|
else:
|
|
|
|
self.assertNotIn('JOIN', sql.upper())
|
|
|
|
spec.inner_joins = old_inner_join
|
|
|
|
self.assertIsNotNone(qr.df)
|
|
|
|
return qr.df
|
|
|
|
|
|
|
|
def test_query_with_expr_groupby_timeseries(self):
|
|
|
|
def cannonicalize_df(df):
|
|
|
|
ret = df.sort_values(by=list(df.columns.values), inplace=False)
|
|
|
|
ret.reset_index(inplace=True, drop=True)
|
|
|
|
return ret
|
|
|
|
|
|
|
|
df1 = self.query_with_expr_helper(is_timeseries=True, inner_join=True)
|
|
|
|
df2 = self.query_with_expr_helper(is_timeseries=True, inner_join=False)
|
|
|
|
self.assertIsNotNone(df2) # df1 can be none if the db does not support join
|
|
|
|
if df1 is not None:
|
|
|
|
pandas.testing.assert_frame_equal(
|
|
|
|
cannonicalize_df(df1),
|
|
|
|
cannonicalize_df(df2))
|
|
|
|
|
|
|
|
def test_query_with_expr_groupby(self):
|
|
|
|
self.query_with_expr_helper(is_timeseries=False)
|
|
|
|
|
2018-07-26 18:20:23 -04:00
|
|
|
def test_sql_mutator(self):
|
|
|
|
tbl = self.get_table_by_name('birth_names')
|
|
|
|
query_obj = dict(
|
|
|
|
groupby=[],
|
|
|
|
metrics=[],
|
|
|
|
filter=[],
|
|
|
|
is_timeseries=False,
|
|
|
|
columns=['name'],
|
|
|
|
granularity=None,
|
|
|
|
from_dttm=None, to_dttm=None,
|
|
|
|
is_prequery=False,
|
|
|
|
extras={},
|
|
|
|
)
|
|
|
|
sql = tbl.get_query_str(query_obj)
|
|
|
|
self.assertNotIn('--COMMENT', sql)
|
|
|
|
|
|
|
|
def mutator(*args):
|
|
|
|
return '--COMMENT\n' + args[0]
|
|
|
|
app.config['SQL_QUERY_MUTATOR'] = mutator
|
|
|
|
sql = tbl.get_query_str(query_obj)
|
|
|
|
self.assertIn('--COMMENT', sql)
|
|
|
|
|
|
|
|
app.config['SQL_QUERY_MUTATOR'] = None
|