mirror of
https://github.com/apache/superset.git
synced 2024-09-12 08:39:45 -04:00
Adding docstrings !
This commit is contained in:
Maxime Beauchemin
parent
3461538fed
commit
ebf55bf20d
@ -36,6 +36,10 @@ QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration'])
|
||||
|
||||
|
||||
class AuditMixinNullable(AuditMixin):
|
||||
|
||||
"""Altering the AuditMixin to use nullable fields, allows creating
|
||||
objects programmatically outside of CRUD"""
|
||||
|
||||
created_on = Column(DateTime, default=datetime.now, nullable=True)
|
||||
changed_on = Column(
|
||||
DateTime, default=datetime.now,
|
||||
@ -142,6 +146,7 @@ class Slice(Model, AuditMixinNullable):
|
||||
|
||||
@property
|
||||
def slice_url(self):
|
||||
"""Defines the url to access the slice"""
|
||||
try:
|
||||
slice_params = json.loads(self.params)
|
||||
except Exception as e:
|
||||
@ -175,7 +180,7 @@ dashboard_slices = Table('dashboard_slices', Model.metadata,
|
||||
|
||||
class Dashboard(Model, AuditMixinNullable):
|
||||
|
||||
"""A dash to slash"""
|
||||
"""The dashboard object!"""
|
||||
|
||||
__tablename__ = 'dashboards'
|
||||
id = Column(Integer, primary_key=True)
|
||||
@ -240,6 +245,9 @@ class Queryable(object):
|
||||
|
||||
|
||||
class Database(Model, AuditMixinNullable):
|
||||
|
||||
"""An ORM object that stores Database related information"""
|
||||
|
||||
__tablename__ = 'dbs'
|
||||
id = Column(Integer, primary_key=True)
|
||||
database_name = Column(String(250), unique=True)
|
||||
@ -256,15 +264,14 @@ class Database(Model, AuditMixinNullable):
|
||||
return self.sqlalchemy_uri
|
||||
|
||||
def grains(self):
|
||||
"""Defines time granularity database-specific expressions.
|
||||
|
||||
"""Defines time granularity database-specific expressions. The idea
|
||||
here is to make it easy for users to change the time grain form a
|
||||
datetime (maybe the source grain is arbitrary timestamps, daily
|
||||
The idea here is to make it easy for users to change the time grain
|
||||
form a datetime (maybe the source grain is arbitrary timestamps, daily
|
||||
or 5 minutes increments) to another, "truncated" datetime. Since
|
||||
each database has slightly different but similar datetime functions,
|
||||
this allows a mapping between database engines and actual functions.
|
||||
"""
|
||||
|
||||
Grain = namedtuple('Grain', 'name function')
|
||||
DB_TIME_GRAINS = {
|
||||
'presto': (
|
||||
@ -314,6 +321,9 @@ class Database(Model, AuditMixinNullable):
|
||||
|
||||
|
||||
class SqlaTable(Model, Queryable, AuditMixinNullable):
|
||||
|
||||
"""An ORM object for SqlAlchemy table references"""
|
||||
|
||||
type = "table"
|
||||
|
||||
__tablename__ = 'tables'
|
||||
@ -554,6 +564,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
|
||||
df=df, duration=datetime.now() - qry_start_dttm, query=sql)
|
||||
|
||||
def fetch_metadata(self):
|
||||
"""Fetches the metadata for the table and merges it in"""
|
||||
table = self.database.get_table(self.table_name)
|
||||
try:
|
||||
table = self.database.get_table(self.table_name)
|
||||
@ -653,6 +664,9 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
|
||||
|
||||
|
||||
class SqlMetric(Model, AuditMixinNullable):
|
||||
|
||||
"""ORM object for metrics, each table can have multiple metrics"""
|
||||
|
||||
__tablename__ = 'sql_metrics'
|
||||
id = Column(Integer, primary_key=True)
|
||||
metric_name = Column(String(512))
|
||||
@ -666,6 +680,9 @@ class SqlMetric(Model, AuditMixinNullable):
|
||||
|
||||
|
||||
class TableColumn(Model, AuditMixinNullable):
|
||||
|
||||
"""ORM object for table columns, each table can have multiple columns"""
|
||||
|
||||
__tablename__ = 'table_columns'
|
||||
id = Column(Integer, primary_key=True)
|
||||
table_id = Column(Integer, ForeignKey('tables.id'))
|
||||
@ -693,6 +710,9 @@ class TableColumn(Model, AuditMixinNullable):
|
||||
|
||||
|
||||
class DruidCluster(Model, AuditMixinNullable):
|
||||
|
||||
"""ORM object referencing the Druid clusters"""
|
||||
|
||||
__tablename__ = 'clusters'
|
||||
id = Column(Integer, primary_key=True)
|
||||
cluster_name = Column(String(250), unique=True)
|
||||
@ -726,6 +746,9 @@ class DruidCluster(Model, AuditMixinNullable):
|
||||
|
||||
|
||||
class DruidDatasource(Model, AuditMixinNullable, Queryable):
|
||||
|
||||
"""ORM object referencing Druid datasources (tables)"""
|
||||
|
||||
type = "druid"
|
||||
|
||||
baselink = "datasourcemodelview"
|
||||
@ -793,6 +816,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
|
||||
][0]
|
||||
|
||||
def latest_metadata(self):
|
||||
"""Returns segment metadata from the latest segment"""
|
||||
client = self.cluster.get_pydruid_client()
|
||||
results = client.time_boundary(datasource=self.datasource_name)
|
||||
if not results:
|
||||
@ -813,6 +837,8 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
|
||||
|
||||
@classmethod
|
||||
def sync_to_db(cls, name, cluster):
|
||||
"""Fetches the metadata for that datasource and merges it into
|
||||
the Panoramix database"""
|
||||
print("Syncing Druid datasource [{}]".format(name))
|
||||
session = get_session()
|
||||
datasource = session.query(cls).filter_by(datasource_name=name).first()
|
||||
@ -855,8 +881,11 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
|
||||
timeseries_limit=None,
|
||||
row_limit=None,
|
||||
inner_from_dttm=None, inner_to_dttm=None,
|
||||
extras=None,
|
||||
extras=None, # noqa
|
||||
select=None):
|
||||
"""Runs a query against Druid and returns a dataframe.
|
||||
This query interface is common to SqlAlchemy and Druid"""
|
||||
# TODO refactor into using a TBD Query object
|
||||
qry_start_dttm = datetime.now()
|
||||
|
||||
inner_from_dttm = inner_from_dttm or from_dttm
|
||||
@ -996,6 +1025,9 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
|
||||
|
||||
|
||||
class Log(Model):
|
||||
|
||||
"""ORM object used to log Panoramix actions to the database"""
|
||||
|
||||
__tablename__ = 'logs'
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
@ -1033,6 +1065,9 @@ class Log(Model):
|
||||
|
||||
|
||||
class DruidMetric(Model):
|
||||
|
||||
"""ORM object referencing Druid metrics for a datasource"""
|
||||
|
||||
__tablename__ = 'metrics'
|
||||
id = Column(Integer, primary_key=True)
|
||||
metric_name = Column(String(512))
|
||||
@ -1055,6 +1090,9 @@ class DruidMetric(Model):
|
||||
|
||||
|
||||
class DruidColumn(Model):
|
||||
|
||||
"""ORM model for storing Druid datasource column metadata"""
|
||||
|
||||
__tablename__ = 'columns'
|
||||
id = Column(Integer, primary_key=True)
|
||||
datasource_name = Column(
|
||||
@ -1080,6 +1118,7 @@ class DruidColumn(Model):
|
||||
return self.type in ('LONG', 'DOUBLE', 'FLOAT')
|
||||
|
||||
def generate_metrics(self):
|
||||
"""Generate metrics based on the column metadata"""
|
||||
M = DruidMetric
|
||||
metrics = []
|
||||
metrics.append(DruidMetric(
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
"""Utility functions used across Panoramix"""
|
||||
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import functools
|
||||
@ -12,10 +14,12 @@ from flask_appbuilder.security.sqla import models as ab_models
|
||||
|
||||
|
||||
class memoized(object):
|
||||
|
||||
"""Decorator that caches a function's return value each time it is called.
|
||||
If called later with the same arguments, the cached value is returned, and
|
||||
not re-evaluated.
|
||||
"""
|
||||
|
||||
def __init__(self, func):
|
||||
self.func = func
|
||||
self.cache = {}
|
||||
@ -47,8 +51,7 @@ def list_minus(l, minus):
|
||||
|
||||
def parse_human_datetime(s):
|
||||
"""
|
||||
Use the parsedatetime lib to return ``datetime.datetime`` from human
|
||||
generated strings
|
||||
Returns ``datetime.datetime`` from human readable strings
|
||||
|
||||
>>> from datetime import date, timedelta
|
||||
>>> from dateutil.relativedelta import relativedelta
|
||||
@ -92,8 +95,7 @@ def merge_perm(sm, permission_name, view_menu_name):
|
||||
|
||||
def parse_human_timedelta(s):
|
||||
"""
|
||||
Use the parsedatetime lib to return ``datetime.datetime`` from human
|
||||
generated strings
|
||||
Returns ``datetime.datetime`` from natural language time deltas
|
||||
|
||||
>>> parse_human_datetime("now") <= datetime.now()
|
||||
True
|
||||
@ -107,7 +109,9 @@ def parse_human_timedelta(s):
|
||||
|
||||
|
||||
class JSONEncodedDict(TypeDecorator):
|
||||
|
||||
"""Represents an immutable structure as a json-encoded string."""
|
||||
|
||||
impl = TEXT
|
||||
def process_bind_param(self, value, dialect):
|
||||
if value is not None:
|
||||
@ -122,6 +126,9 @@ class JSONEncodedDict(TypeDecorator):
|
||||
|
||||
|
||||
class ColorFactory(object):
|
||||
|
||||
"""Used to generated arrays of colors server side"""
|
||||
|
||||
BNB_COLORS = [
|
||||
#rausch hackb kazan babu lima beach barol
|
||||
'#ff5a5f', '#7b0051', '#007A87', '#00d1c1', '#8ce071', '#ffb400', '#b4a76c',
|
||||
@ -134,7 +141,8 @@ class ColorFactory(object):
|
||||
self.hash_based = hash_based
|
||||
|
||||
def get(self, s):
|
||||
"""
|
||||
"""Gets a color from a string and memoize the association
|
||||
|
||||
>>> cf = ColorFactory()
|
||||
>>> cf.get('item_1')
|
||||
'#ff5a5f'
|
||||
@ -155,9 +163,7 @@ class ColorFactory(object):
|
||||
|
||||
|
||||
def init(panoramix):
|
||||
"""
|
||||
Inits the Panoramix application with security roles and such
|
||||
"""
|
||||
"""Inits the Panoramix application with security roles and such"""
|
||||
db = panoramix.db
|
||||
models = panoramix.models
|
||||
sm = panoramix.appbuilder.sm
|
||||
@ -204,9 +210,7 @@ def init(panoramix):
|
||||
|
||||
|
||||
def datetime_f(dttm):
|
||||
"""
|
||||
Formats datetime to take less room is recent
|
||||
"""
|
||||
"""Formats datetime to take less room when it is recent"""
|
||||
if dttm:
|
||||
dttm = dttm.isoformat()
|
||||
now_iso = datetime.now().isoformat()
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
"""Flask web views for Panoramix"""
|
||||
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
@ -44,7 +46,7 @@ class PanoramixModelView(ModelView):
|
||||
page_size = 500
|
||||
|
||||
|
||||
class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView): # noqa
|
||||
datamodel = SQLAInterface(models.TableColumn)
|
||||
can_delete = False
|
||||
edit_columns = [
|
||||
@ -72,7 +74,8 @@ appbuilder.add_link(
|
||||
|
||||
appbuilder.add_separator("Sources")
|
||||
|
||||
class DruidColumnInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
|
||||
class DruidColumnInlineView(CompactCRUDMixin, PanoramixModelView): # noqa
|
||||
datamodel = SQLAInterface(models.DruidColumn)
|
||||
edit_columns = [
|
||||
'column_name', 'description', 'datasource', 'groupby',
|
||||
@ -89,7 +92,7 @@ class DruidColumnInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
appbuilder.add_view_no_menu(DruidColumnInlineView)
|
||||
|
||||
|
||||
class SqlMetricInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
class SqlMetricInlineView(CompactCRUDMixin, PanoramixModelView): # noqa
|
||||
datamodel = SQLAInterface(models.SqlMetric)
|
||||
list_columns = ['metric_name', 'verbose_name', 'metric_type']
|
||||
edit_columns = [
|
||||
@ -100,7 +103,7 @@ class SqlMetricInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
appbuilder.add_view_no_menu(SqlMetricInlineView)
|
||||
|
||||
|
||||
class DruidMetricInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
class DruidMetricInlineView(CompactCRUDMixin, PanoramixModelView): # noqa
|
||||
datamodel = SQLAInterface(models.DruidMetric)
|
||||
list_columns = ['metric_name', 'verbose_name', 'metric_type']
|
||||
edit_columns = [
|
||||
@ -115,7 +118,7 @@ class DruidMetricInlineView(CompactCRUDMixin, PanoramixModelView):
|
||||
appbuilder.add_view_no_menu(DruidMetricInlineView)
|
||||
|
||||
|
||||
class DatabaseView(PanoramixModelView, DeleteMixin):
|
||||
class DatabaseView(PanoramixModelView, DeleteMixin): # noqa
|
||||
datamodel = SQLAInterface(models.Database)
|
||||
list_columns = ['database_name', 'sql_link', 'created_by_', 'changed_on']
|
||||
order_columns = utils.list_minus(list_columns, ['created_by_'])
|
||||
@ -149,7 +152,7 @@ appbuilder.add_view(
|
||||
category_icon='fa-database',)
|
||||
|
||||
|
||||
class TableModelView(PanoramixModelView, DeleteMixin):
|
||||
class TableModelView(PanoramixModelView, DeleteMixin): # noqa
|
||||
datamodel = SQLAInterface(models.SqlaTable)
|
||||
list_columns = [
|
||||
'table_link', 'database', 'sql_link', 'is_featured',
|
||||
@ -191,7 +194,7 @@ appbuilder.add_view(
|
||||
appbuilder.add_separator("Sources")
|
||||
|
||||
|
||||
class DruidClusterModelView(PanoramixModelView, DeleteMixin):
|
||||
class DruidClusterModelView(PanoramixModelView, DeleteMixin): # noqa
|
||||
datamodel = SQLAInterface(models.DruidCluster)
|
||||
add_columns = [
|
||||
'cluster_name',
|
||||
@ -209,7 +212,7 @@ appbuilder.add_view(
|
||||
category_icon='fa-database',)
|
||||
|
||||
|
||||
class SliceModelView(PanoramixModelView, DeleteMixin):
|
||||
class SliceModelView(PanoramixModelView, DeleteMixin): # noqa
|
||||
datamodel = SQLAInterface(models.Slice)
|
||||
can_add = False
|
||||
list_columns = [
|
||||
@ -237,7 +240,7 @@ appbuilder.add_view(
|
||||
category_icon='',)
|
||||
|
||||
|
||||
class DashboardModelView(PanoramixModelView, DeleteMixin):
|
||||
class DashboardModelView(PanoramixModelView, DeleteMixin): # noqa
|
||||
datamodel = SQLAInterface(models.Dashboard)
|
||||
list_columns = ['dashboard_link', 'created_by_', 'changed_on']
|
||||
order_columns = utils.list_minus(list_columns, ['created_by_'])
|
||||
@ -289,7 +292,7 @@ appbuilder.add_view(
|
||||
icon="fa-list-ol")
|
||||
|
||||
|
||||
class DruidDatasourceModelView(PanoramixModelView, DeleteMixin):
|
||||
class DruidDatasourceModelView(PanoramixModelView, DeleteMixin): # noqa
|
||||
datamodel = SQLAInterface(models.DruidDatasource)
|
||||
list_columns = [
|
||||
'datasource_link', 'cluster', 'owner',
|
||||
@ -362,6 +365,7 @@ appbuilder.add_view_no_menu(R)
|
||||
|
||||
|
||||
class Panoramix(BaseView):
|
||||
"""The base views for Panoramix!"""
|
||||
|
||||
@has_access
|
||||
@expose("/explore/<datasource_type>/<datasource_id>/")
|
||||
@ -502,6 +506,7 @@ class Panoramix(BaseView):
|
||||
@has_access
|
||||
@expose("/checkbox/<model_view>/<id_>/<attr>/<value>", methods=['GET'])
|
||||
def checkbox(self, model_view, id_, attr, value):
|
||||
"""endpoint for checking/unchecking any boolean in a sqla model"""
|
||||
model = None
|
||||
if model_view == 'TableColumnInlineView':
|
||||
model = models.TableColumn
|
||||
@ -518,6 +523,7 @@ class Panoramix(BaseView):
|
||||
@has_access
|
||||
@expose("/save_dash/<dashboard_id>/", methods=['GET', 'POST'])
|
||||
def save_dash(self, dashboard_id):
|
||||
"""Save a dashboard's metadata"""
|
||||
data = json.loads(request.form.get('data'))
|
||||
positions = data['positions']
|
||||
slice_ids = [int(d['slice_id']) for d in positions]
|
||||
@ -540,6 +546,7 @@ class Panoramix(BaseView):
|
||||
@has_access
|
||||
@expose("/testconn", methods=["POST", "GET"])
|
||||
def testconn(self):
|
||||
"""Tests a sqla connection"""
|
||||
try:
|
||||
uri = request.form.get('uri')
|
||||
engine = create_engine(uri)
|
||||
@ -554,6 +561,7 @@ class Panoramix(BaseView):
|
||||
@has_access
|
||||
@expose("/dashboard/<dashboard_id>/")
|
||||
def dashboard(self, dashboard_id):
|
||||
"""Server side rendering for a dashboard"""
|
||||
session = db.session()
|
||||
qry = session.query(models.Dashboard)
|
||||
if dashboard_id.isdigit():
|
||||
|
||||
Loading…
Reference in New Issue
Block a user