From 39a9b3b1e98f9f10d1d39f60ced56e4655fc3f88 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 23 Aug 2015 14:57:08 -0700 Subject: [PATCH] Better automation around database table semantic layer --- app/models.py | 75 ++++++++++++++++++++++++-- app/templates/panoramix/viz_table.html | 44 +++++++-------- app/views.py | 4 +- app/viz.py | 4 +- 4 files changed, 99 insertions(+), 28 deletions(-) diff --git a/app/models.py b/app/models.py index 37c1c4a6e4..391f195c63 100644 --- a/app/models.py +++ b/app/models.py @@ -1,6 +1,7 @@ from flask.ext.appbuilder import Model from datetime import timedelta from flask.ext.appbuilder.models.mixins import AuditMixin +from flask import request, redirect, flash, Response from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean, DateTime from sqlalchemy import create_engine, MetaData, desc from sqlalchemy import Table as sqlaTable @@ -68,7 +69,7 @@ class Table(Model, Queryable, AuditMixin): main_datetime_column = relationship( 'TableColumn', foreign_keys=[main_datetime_column_id]) default_endpoint = Column(Text) - database_id = Column(Integer, ForeignKey('dbs.id')) + database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) database = relationship( 'Database', backref='tables', foreign_keys=[database_id]) @@ -261,8 +262,18 @@ class Table(Model, Queryable, AuditMixin): def fetch_metadata(self): - table = self.database.get_table(self.table_name) + try: + table = self.database.get_table(self.table_name) + except Exception as e: + flash( + "Table doesn't see to exist in the specified database, " + "couldn't fetch column information", "danger") + return + TC = TableColumn + M = SqlMetric + metrics = [] + any_date_col = None for col in table.columns: dbcol = ( db.session @@ -274,14 +285,70 @@ class Table(Model, Queryable, AuditMixin): db.session.flush() if not dbcol: dbcol = TableColumn(column_name=col.name) - if str(col.type) in ('VARCHAR', 'STRING'): + if ( + str(col.type).startswith('VARCHAR') or + str(col.type).startswith('STRING')): dbcol.groupby = True dbcol.filterable = True - self.columns.append(dbcol) + db.session.merge(self) + self.columns.append(dbcol) + if not any_date_col and 'date' in str(col.type).lower(): + any_date_col = dbcol + + if dbcol.sum: + metrics.append(M( + metric_name='sum__' + dbcol.column_name, + verbose_name='sum__' + dbcol.column_name, + metric_type='sum', + expression="SUM({})".format(dbcol.column_name) + )) + if dbcol.max: + metrics.append(M( + metric_name='max__' + dbcol.column_name, + verbose_name='max__' + dbcol.column_name, + metric_type='max', + expression="MAX({})".format(dbcol.column_name) + )) + if dbcol.min: + metrics.append(M( + metric_name='min__' + dbcol.column_name, + verbose_name='min__' + dbcol.column_name, + metric_type='min', + expression="MIN({})".format(dbcol.column_name) + )) + if dbcol.count_distinct: + metrics.append(M( + metric_name='count_distinct__' + dbcol.column_name, + verbose_name='count_distinct__' + dbcol.column_name, + metric_type='count_distinct', + expression="COUNT(DISTINCT {})".format(dbcol.column_name) + )) dbcol.type = str(col.type) + db.session.merge(self) db.session.commit() + metrics.append(M( + metric_name='count', + verbose_name='COUNT(*)', + metric_type='count', + expression="COUNT(*)" + )) + for metric in metrics: + m = ( + db.session.query(M) + .filter(M.metric_name==metric.metric_name) + .filter(M.table==self) + .first() + ) + metric.table = self + if not m: + db.session.add(metric) + db.session.commit() + if not self.main_datetime_column: + self.main_datetime_column = any_date_col + + class SqlMetric(Model, AuditMixin): diff --git a/app/templates/panoramix/viz_table.html b/app/templates/panoramix/viz_table.html index e78c94fcd0..18fc6e122f 100644 --- a/app/templates/panoramix/viz_table.html +++ b/app/templates/panoramix/viz_table.html @@ -7,28 +7,30 @@ {% block viz %} {{ super() }} - - - - {% for col in df.columns if not col.endswith('__perc') %} - + {% if not error_msg %} +
{{ col }}
+ + + {% for col in df.columns if not col.endswith('__perc') %} + + {% endfor %} + + + + {% for row in df.to_dict(orient="records") %} + + {% for col in df.columns if not col.endswith('__perc') %} + {% if col + '__perc' in df.columns %} + + {% else %} + + {% endif %} + {% endfor %} + {% endfor %} - - - - {% for row in df.to_dict(orient="records") %} - - {% for col in df.columns if not col.endswith('__perc') %} - {% if col + '__perc' in df.columns %} - - {% else %} - - {% endif %} - {% endfor %} - - {% endfor %} - -
{{ col }}
{{ row[col] }}{{ row[col] }}
{{ row[col] }}{{ row[col] }}
+ + + {% endif %} {% endblock %} {% block extra_fields %} diff --git a/app/views.py b/app/views.py index 3e6cb15416..a2af424077 100644 --- a/app/views.py +++ b/app/views.py @@ -124,7 +124,7 @@ class TableView(ModelView, DeleteMixin): edit_columns = ['table_name', 'database', 'main_datetime_column', 'default_endpoint'] related_views = [TableColumnInlineView, SqlMetricInlineView] - def post_insert(self, table): + def post_add(self, table): table.fetch_metadata() def post_update(self, table): @@ -147,7 +147,7 @@ class DatasourceModelView(ModelView, DeleteMixin): page_size = 100 base_order = ('datasource_name', 'asc') - def post_insert(self, datasource): + def post_add(self, datasource): datasource.generate_metrics() def post_update(self, datasource): diff --git a/app/viz.py b/app/viz.py index 8ecf5702b2..60b4e5ac68 100644 --- a/app/viz.py +++ b/app/viz.py @@ -6,6 +6,7 @@ from app import utils from app.highchart import Highchart, HighchartBubble from wtforms import Form, SelectMultipleField, SelectField, TextField import config +import logging from pydruid.utils.filters import Dimension, Filter @@ -87,6 +88,7 @@ class BaseViz(object): self.df_prep() self.form_prep() except Exception as e: + logging.exception(e) self.error_msg = str(e) @@ -171,7 +173,7 @@ class TableViz(BaseViz): df = self.df row_limit = request.args.get("row_limit") if df is None or df.empty: - flash("No data.", "error") + return super(TableViz, self).render(error_msg="No data.") else: if self.form_data.get("granularity") == "all" and 'timestamp' in df: del df['timestamp']