mirror of https://github.com/apache/superset.git
Refactoring more in the connector base classes (#2431)
This commit is contained in:
parent
398036d77e
commit
121b1d0951
|
@ -12,6 +12,7 @@ from __future__ import unicode_literals
|
||||||
import imp
|
import imp
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
from dateutil import tz
|
from dateutil import tz
|
||||||
from flask_appbuilder.security.manager import AUTH_DB
|
from flask_appbuilder.security.manager import AUTH_DB
|
||||||
|
@ -178,10 +179,10 @@ DRUID_DATA_SOURCE_BLACKLIST = []
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# Modules, datasources and middleware to be registered
|
# Modules, datasources and middleware to be registered
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
DEFAULT_MODULE_DS_MAP = {
|
DEFAULT_MODULE_DS_MAP = OrderedDict([
|
||||||
'superset.connectors.druid.models': ['DruidDatasource'],
|
('superset.connectors.sqla.models', ['SqlaTable']),
|
||||||
'superset.connectors.sqla.models': ['SqlaTable'],
|
('superset.connectors.druid.models', ['DruidDatasource']),
|
||||||
}
|
])
|
||||||
ADDITIONAL_MODULE_DS_MAP = {}
|
ADDITIONAL_MODULE_DS_MAP = {}
|
||||||
ADDITIONAL_MIDDLEWARE = []
|
ADDITIONAL_MIDDLEWARE = []
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from sqlalchemy import Column, Integer, String, Text, Boolean
|
from sqlalchemy import (
|
||||||
|
Column, Integer, String, Text, Boolean,
|
||||||
|
)
|
||||||
from superset import utils
|
from superset import utils
|
||||||
from superset.models.helpers import AuditMixinNullable, ImportMixin
|
from superset.models.helpers import AuditMixinNullable, ImportMixin
|
||||||
|
|
||||||
|
@ -12,9 +13,23 @@ class BaseDatasource(AuditMixinNullable, ImportMixin):
|
||||||
|
|
||||||
__tablename__ = None # {connector_name}_datasource
|
__tablename__ = None # {connector_name}_datasource
|
||||||
|
|
||||||
|
column_class = None # link to derivative of BaseColumn
|
||||||
|
metric_class = None # link to derivative of BaseMetric
|
||||||
|
|
||||||
# Used to do code highlighting when displaying the query in the UI
|
# Used to do code highlighting when displaying the query in the UI
|
||||||
query_language = None
|
query_language = None
|
||||||
|
|
||||||
|
# Columns
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
description = Column(Text)
|
||||||
|
default_endpoint = Column(Text)
|
||||||
|
is_featured = Column(Boolean, default=False)
|
||||||
|
filter_select_enabled = Column(Boolean, default=False)
|
||||||
|
offset = Column(Integer, default=0)
|
||||||
|
cache_timeout = Column(Integer)
|
||||||
|
params = Column(String(1000))
|
||||||
|
perm = Column(String(1000))
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def column_names(self):
|
def column_names(self):
|
||||||
return sorted([c.column_name for c in self.columns])
|
return sorted([c.column_name for c in self.columns])
|
||||||
|
|
|
@ -312,38 +312,29 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
|
|
||||||
"""ORM object referencing Druid datasources (tables)"""
|
"""ORM object referencing Druid datasources (tables)"""
|
||||||
|
|
||||||
|
__tablename__ = 'datasources'
|
||||||
|
|
||||||
type = "druid"
|
type = "druid"
|
||||||
query_langtage = "json"
|
query_langtage = "json"
|
||||||
metric_class = DruidMetric
|
|
||||||
cluster_class = DruidCluster
|
cluster_class = DruidCluster
|
||||||
|
metric_class = DruidMetric
|
||||||
|
column_class = DruidColumn
|
||||||
|
|
||||||
baselink = "druiddatasourcemodelview"
|
baselink = "druiddatasourcemodelview"
|
||||||
|
|
||||||
__tablename__ = 'datasources'
|
# Columns
|
||||||
id = Column(Integer, primary_key=True)
|
|
||||||
datasource_name = Column(String(255), unique=True)
|
datasource_name = Column(String(255), unique=True)
|
||||||
is_featured = Column(Boolean, default=False)
|
|
||||||
is_hidden = Column(Boolean, default=False)
|
is_hidden = Column(Boolean, default=False)
|
||||||
filter_select_enabled = Column(Boolean, default=False)
|
|
||||||
description = Column(Text)
|
|
||||||
fetch_values_from = Column(String(100))
|
fetch_values_from = Column(String(100))
|
||||||
default_endpoint = Column(Text)
|
cluster_name = Column(
|
||||||
|
String(250), ForeignKey('clusters.cluster_name'))
|
||||||
|
cluster = relationship(
|
||||||
|
'DruidCluster', backref='datasources', foreign_keys=[cluster_name])
|
||||||
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
||||||
owner = relationship(
|
owner = relationship(
|
||||||
'User',
|
'User',
|
||||||
backref=backref('datasources', cascade='all, delete-orphan'),
|
backref=backref('datasources', cascade='all, delete-orphan'),
|
||||||
foreign_keys=[user_id])
|
foreign_keys=[user_id])
|
||||||
cluster_name = Column(
|
|
||||||
String(250), ForeignKey('clusters.cluster_name'))
|
|
||||||
cluster = relationship(
|
|
||||||
'DruidCluster', backref='datasources', foreign_keys=[cluster_name])
|
|
||||||
offset = Column(Integer, default=0)
|
|
||||||
cache_timeout = Column(Integer)
|
|
||||||
params = Column(String(1000))
|
|
||||||
perm = Column(String(1000))
|
|
||||||
|
|
||||||
metric_cls = DruidMetric
|
|
||||||
column_cls = DruidColumn
|
|
||||||
|
|
||||||
export_fields = (
|
export_fields = (
|
||||||
'datasource_name', 'is_hidden', 'description', 'default_endpoint',
|
'datasource_name', 'is_hidden', 'description', 'default_endpoint',
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
|
from datetime import datetime
|
||||||
|
import logging
|
||||||
|
|
||||||
import sqlalchemy as sqla
|
import sqlalchemy as sqla
|
||||||
|
|
||||||
from flask import Markup
|
from flask import Markup, flash, redirect
|
||||||
from flask_appbuilder import CompactCRUDMixin
|
from flask_appbuilder import CompactCRUDMixin, expose
|
||||||
from flask_appbuilder.models.sqla.interface import SQLAInterface
|
from flask_appbuilder.models.sqla.interface import SQLAInterface
|
||||||
|
|
||||||
from flask_babel import lazy_gettext as _
|
from flask_babel import lazy_gettext as _
|
||||||
|
@ -9,6 +12,9 @@ from flask_babel import gettext as __
|
||||||
|
|
||||||
import superset
|
import superset
|
||||||
from superset import db, utils, appbuilder, sm, security
|
from superset import db, utils, appbuilder, sm, security
|
||||||
|
from superset.connectors.connector_registry import ConnectorRegistry
|
||||||
|
from superset.utils import has_access
|
||||||
|
from superset.views.base import BaseSupersetView
|
||||||
from superset.views.base import (
|
from superset.views.base import (
|
||||||
SupersetModelView, validate_json, DeleteMixin, ListWidgetWithCheckboxes,
|
SupersetModelView, validate_json, DeleteMixin, ListWidgetWithCheckboxes,
|
||||||
DatasourceFilter, get_datasource_exist_error_mgs)
|
DatasourceFilter, get_datasource_exist_error_mgs)
|
||||||
|
@ -205,3 +211,46 @@ appbuilder.add_view(
|
||||||
category="Sources",
|
category="Sources",
|
||||||
category_label=__("Sources"),
|
category_label=__("Sources"),
|
||||||
icon="fa-cube")
|
icon="fa-cube")
|
||||||
|
|
||||||
|
|
||||||
|
class Druid(BaseSupersetView):
|
||||||
|
"""The base views for Superset!"""
|
||||||
|
|
||||||
|
@has_access
|
||||||
|
@expose("/refresh_datasources/")
|
||||||
|
def refresh_datasources(self):
|
||||||
|
"""endpoint that refreshes druid datasources metadata"""
|
||||||
|
session = db.session()
|
||||||
|
DruidCluster = ConnectorRegistry.sources['druid'].cluster_class
|
||||||
|
for cluster in session.query(DruidCluster).all():
|
||||||
|
cluster_name = cluster.cluster_name
|
||||||
|
try:
|
||||||
|
cluster.refresh_datasources()
|
||||||
|
except Exception as e:
|
||||||
|
flash(
|
||||||
|
"Error while processing cluster '{}'\n{}".format(
|
||||||
|
cluster_name, utils.error_msg_from_exception(e)),
|
||||||
|
"danger")
|
||||||
|
logging.exception(e)
|
||||||
|
return redirect('/druidclustermodelview/list/')
|
||||||
|
cluster.metadata_last_refreshed = datetime.now()
|
||||||
|
flash(
|
||||||
|
"Refreshed metadata from cluster "
|
||||||
|
"[" + cluster.cluster_name + "]",
|
||||||
|
'info')
|
||||||
|
session.commit()
|
||||||
|
return redirect("/druiddatasourcemodelview/list/")
|
||||||
|
|
||||||
|
appbuilder.add_view_no_menu(Druid)
|
||||||
|
|
||||||
|
appbuilder.add_link(
|
||||||
|
"Refresh Druid Metadata",
|
||||||
|
label=__("Refresh Druid Metadata"),
|
||||||
|
href='/druid/refresh_datasources/',
|
||||||
|
category='Sources',
|
||||||
|
category_label=__("Sources"),
|
||||||
|
category_icon='fa-database',
|
||||||
|
icon="fa-cog")
|
||||||
|
|
||||||
|
|
||||||
|
appbuilder.add_separator("Sources", )
|
||||||
|
|
|
@ -162,33 +162,26 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
type = "table"
|
type = "table"
|
||||||
query_language = 'sql'
|
query_language = 'sql'
|
||||||
metric_class = SqlMetric
|
metric_class = SqlMetric
|
||||||
|
column_class = TableColumn
|
||||||
|
|
||||||
__tablename__ = 'tables'
|
__tablename__ = 'tables'
|
||||||
id = Column(Integer, primary_key=True)
|
|
||||||
table_name = Column(String(250))
|
table_name = Column(String(250))
|
||||||
main_dttm_col = Column(String(250))
|
main_dttm_col = Column(String(250))
|
||||||
description = Column(Text)
|
|
||||||
default_endpoint = Column(Text)
|
|
||||||
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
|
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
|
||||||
is_featured = Column(Boolean, default=False)
|
|
||||||
filter_select_enabled = Column(Boolean, default=False)
|
|
||||||
fetch_values_predicate = Column(String(1000))
|
fetch_values_predicate = Column(String(1000))
|
||||||
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
||||||
owner = relationship('User', backref='tables', foreign_keys=[user_id])
|
owner = relationship(
|
||||||
|
'User',
|
||||||
|
backref='tables',
|
||||||
|
foreign_keys=[user_id])
|
||||||
database = relationship(
|
database = relationship(
|
||||||
'Database',
|
'Database',
|
||||||
backref=backref('tables', cascade='all, delete-orphan'),
|
backref=backref('tables', cascade='all, delete-orphan'),
|
||||||
foreign_keys=[database_id])
|
foreign_keys=[database_id])
|
||||||
offset = Column(Integer, default=0)
|
|
||||||
cache_timeout = Column(Integer)
|
|
||||||
schema = Column(String(255))
|
schema = Column(String(255))
|
||||||
sql = Column(Text)
|
sql = Column(Text)
|
||||||
params = Column(Text)
|
|
||||||
perm = Column(String(1000))
|
|
||||||
|
|
||||||
baselink = "tablemodelview"
|
baselink = "tablemodelview"
|
||||||
column_cls = TableColumn
|
|
||||||
metric_cls = SqlMetric
|
|
||||||
export_fields = (
|
export_fields = (
|
||||||
'table_name', 'main_dttm_col', 'description', 'default_endpoint',
|
'table_name', 'main_dttm_col', 'description', 'default_endpoint',
|
||||||
'database_id', 'is_featured', 'offset', 'cache_timeout', 'schema',
|
'database_id', 'is_featured', 'offset', 'cache_timeout', 'schema',
|
||||||
|
|
|
@ -38,7 +38,7 @@ def import_datasource(
|
||||||
new_m.table_id = datasource.id
|
new_m.table_id = datasource.id
|
||||||
logging.info('Importing metric {} from the datasource: {}'.format(
|
logging.info('Importing metric {} from the datasource: {}'.format(
|
||||||
new_m.to_json(), i_datasource.full_name))
|
new_m.to_json(), i_datasource.full_name))
|
||||||
imported_m = i_datasource.metric_cls.import_obj(new_m)
|
imported_m = i_datasource.metric_class.import_obj(new_m)
|
||||||
if (imported_m.metric_name not in
|
if (imported_m.metric_name not in
|
||||||
[m.metric_name for m in datasource.metrics]):
|
[m.metric_name for m in datasource.metrics]):
|
||||||
datasource.metrics.append(imported_m)
|
datasource.metrics.append(imported_m)
|
||||||
|
@ -48,7 +48,7 @@ def import_datasource(
|
||||||
new_c.table_id = datasource.id
|
new_c.table_id = datasource.id
|
||||||
logging.info('Importing column {} from the datasource: {}'.format(
|
logging.info('Importing column {} from the datasource: {}'.format(
|
||||||
new_c.to_json(), i_datasource.full_name))
|
new_c.to_json(), i_datasource.full_name))
|
||||||
imported_c = i_datasource.column_cls.import_obj(new_c)
|
imported_c = i_datasource.column_class.import_obj(new_c)
|
||||||
if (imported_c.column_name not in
|
if (imported_c.column_name not in
|
||||||
[c.column_name for c in datasource.columns]):
|
[c.column_name for c in datasource.columns]):
|
||||||
datasource.columns.append(imported_c)
|
datasource.columns.append(imported_c)
|
||||||
|
|
|
@ -9,7 +9,6 @@ import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import pickle
|
import pickle
|
||||||
import re
|
import re
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
import zlib
|
import zlib
|
||||||
|
@ -130,7 +129,7 @@ def check_ownership(obj, raise_if_false=True):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
security_exception = utils.SupersetSecurityException(
|
security_exception = utils.SupersetSecurityException(
|
||||||
"You don't have the rights to alter [{}]".format(obj))
|
"You don't have the rights to alter [{}]".format(obj))
|
||||||
|
|
||||||
if g.user.is_anonymous():
|
if g.user.is_anonymous():
|
||||||
if raise_if_false:
|
if raise_if_false:
|
||||||
|
@ -762,8 +761,8 @@ class Superset(BaseSupersetView):
|
||||||
granted_perms = []
|
granted_perms = []
|
||||||
for datasource in datasources:
|
for datasource in datasources:
|
||||||
view_menu_perm = sm.find_permission_view_menu(
|
view_menu_perm = sm.find_permission_view_menu(
|
||||||
view_menu_name=datasource.perm,
|
view_menu_name=datasource.perm,
|
||||||
permission_name='datasource_access')
|
permission_name='datasource_access')
|
||||||
# prevent creating empty permissions
|
# prevent creating empty permissions
|
||||||
if view_menu_perm and view_menu_perm.view_menu:
|
if view_menu_perm and view_menu_perm.view_menu:
|
||||||
role.permissions.append(view_menu_perm)
|
role.permissions.append(view_menu_perm)
|
||||||
|
@ -1214,8 +1213,12 @@ class Superset(BaseSupersetView):
|
||||||
@expose("/checkbox/<model_view>/<id_>/<attr>/<value>", methods=['GET'])
|
@expose("/checkbox/<model_view>/<id_>/<attr>/<value>", methods=['GET'])
|
||||||
def checkbox(self, model_view, id_, attr, value):
|
def checkbox(self, model_view, id_, attr, value):
|
||||||
"""endpoint for checking/unchecking any boolean in a sqla model"""
|
"""endpoint for checking/unchecking any boolean in a sqla model"""
|
||||||
Col = ConnectorRegistry.sources['table'].column_cls
|
modelview_to_model = {
|
||||||
obj = db.session.query(Col).filter_by(id=id_).first()
|
'TableColumnInlineView':
|
||||||
|
ConnectorRegistry.sources['table'].column_class,
|
||||||
|
}
|
||||||
|
model = modelview_to_model[model_view]
|
||||||
|
obj = db.session.query(model).filter_by(id=id_).first()
|
||||||
if obj:
|
if obj:
|
||||||
setattr(obj, attr, value == 'true')
|
setattr(obj, attr, value == 'true')
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
@ -1750,6 +1753,7 @@ class Superset(BaseSupersetView):
|
||||||
@expose("/sqllab_viz/", methods=['POST'])
|
@expose("/sqllab_viz/", methods=['POST'])
|
||||||
@log_this
|
@log_this
|
||||||
def sqllab_viz(self):
|
def sqllab_viz(self):
|
||||||
|
SqlaTable = ConnectorRegistry.sources['table']
|
||||||
data = json.loads(request.form.get('data'))
|
data = json.loads(request.form.get('data'))
|
||||||
table_name = data.get('datasourceName')
|
table_name = data.get('datasourceName')
|
||||||
viz_type = data.get('chartType')
|
viz_type = data.get('chartType')
|
||||||
|
@ -2158,32 +2162,6 @@ class Superset(BaseSupersetView):
|
||||||
status=200,
|
status=200,
|
||||||
mimetype="application/json")
|
mimetype="application/json")
|
||||||
|
|
||||||
@has_access
|
|
||||||
@expose("/refresh_datasources/")
|
|
||||||
def refresh_datasources(self):
|
|
||||||
"""endpoint that refreshes druid datasources metadata"""
|
|
||||||
session = db.session()
|
|
||||||
DruidDatasource = ConnectorRegistry.sources['druid']
|
|
||||||
DruidCluster = DruidDatasource.cluster_class
|
|
||||||
for cluster in session.query(DruidCluster).all():
|
|
||||||
cluster_name = cluster.cluster_name
|
|
||||||
try:
|
|
||||||
cluster.refresh_datasources()
|
|
||||||
except Exception as e:
|
|
||||||
flash(
|
|
||||||
"Error while processing cluster '{}'\n{}".format(
|
|
||||||
cluster_name, utils.error_msg_from_exception(e)),
|
|
||||||
"danger")
|
|
||||||
logging.exception(e)
|
|
||||||
return redirect('/druidclustermodelview/list/')
|
|
||||||
cluster.metadata_last_refreshed = datetime.now()
|
|
||||||
flash(
|
|
||||||
"Refreshed metadata from cluster "
|
|
||||||
"[" + cluster.cluster_name + "]",
|
|
||||||
'info')
|
|
||||||
session.commit()
|
|
||||||
return redirect("/druiddatasourcemodelview/list/")
|
|
||||||
|
|
||||||
@app.errorhandler(500)
|
@app.errorhandler(500)
|
||||||
def show_traceback(self):
|
def show_traceback(self):
|
||||||
return render_template(
|
return render_template(
|
||||||
|
@ -2257,16 +2235,6 @@ class Superset(BaseSupersetView):
|
||||||
)
|
)
|
||||||
appbuilder.add_view_no_menu(Superset)
|
appbuilder.add_view_no_menu(Superset)
|
||||||
|
|
||||||
if config['DRUID_IS_ACTIVE']:
|
|
||||||
appbuilder.add_link(
|
|
||||||
"Refresh Druid Metadata",
|
|
||||||
label=__("Refresh Druid Metadata"),
|
|
||||||
href='/superset/refresh_datasources/',
|
|
||||||
category='Sources',
|
|
||||||
category_label=__("Sources"),
|
|
||||||
category_icon='fa-database',
|
|
||||||
icon="fa-cog")
|
|
||||||
|
|
||||||
|
|
||||||
class CssTemplateModelView(SupersetModelView, DeleteMixin):
|
class CssTemplateModelView(SupersetModelView, DeleteMixin):
|
||||||
datamodel = SQLAInterface(models.CssTemplate)
|
datamodel = SQLAInterface(models.CssTemplate)
|
||||||
|
|
Loading…
Reference in New Issue