diff --git a/panoramix/TODO.md b/panoramix/TODO.md index 192021631f..cc31f88ea6 100644 --- a/panoramix/TODO.md +++ b/panoramix/TODO.md @@ -1,3 +1,3 @@ # TODO -* Default URL params per datasource -* Get config metrics to work +* Multi-filters +* multi-metrics diff --git a/panoramix/app/models.py b/panoramix/app/models.py index b9b81d29e6..39259323ac 100644 --- a/panoramix/app/models.py +++ b/panoramix/app/models.py @@ -5,14 +5,9 @@ from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean from sqlalchemy.orm import relationship from app import db, utils from dateutil.parser import parse -""" - -You can use the extra Flask-AppBuilder fields and Mixin's - -AuditMixin will add automatic timestamp of created and modified by who +import json -""" client = utils.get_pydruid_client() class Datasource(Model, AuditMixin): @@ -22,12 +17,11 @@ class Datasource(Model, AuditMixin): is_featured = Column(Boolean, default=False) is_hidden = Column(Boolean, default=False) description = Column(Text) - columns = relationship('Column', backref='datasource') - udfs = relationship('JavascriptUdf', backref='datasource') + default_endpoint = Column(Text) @property - def metrics(self): - return [col.column_name for col in self.columns if not col.groupby] + def metrics_combo(self): + return [(m.metric_name, m.verbose_name) for m in self.metrics] def __repr__(self): return self.datasource_name @@ -37,9 +31,18 @@ class Datasource(Model, AuditMixin): url = "/panoramix/datasource/{}/".format(self.datasource_name) return '{self.datasource_name}'.format(**locals()) + def get_metric_obj(self, metric_name): + return [ + m.json_obj for m in self.metrics + if m.metric_name == metric_name + ][0] + @classmethod def latest_metadata(cls, name): results = client.time_boundary(datasource=name) + print "---" * 100 + print name + print results max_time = results[0]['result']['maxTime'] max_time = parse(max_time) intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' @@ -47,7 +50,13 @@ class Datasource(Model, AuditMixin): segment_metadata = client.segment_metadata( datasource=name, intervals=intervals) - return segment_metadata[-1]['columns'] + print segment_metadata + if segment_metadata: + return segment_metadata[-1]['columns'] + + def generate_metrics(self): + for col in self.columns: + col.generate_metrics() @classmethod def sync_to_db(cls, name): @@ -55,6 +64,8 @@ class Datasource(Model, AuditMixin): if not datasource: db.session.add(cls(datasource_name=name)) cols = cls.latest_metadata(name) + if not cols: + return for col in cols: col_obj = ( db.session @@ -71,7 +82,7 @@ class Datasource(Model, AuditMixin): col_obj.filterable = True if col_obj: col_obj.type = cols[col]['type'] - + col_obj.generate_metrics() db.session.commit() @property @@ -87,19 +98,21 @@ class Datasource(Model, AuditMixin): return sorted([c.column_name for c in self.columns if c.filterable]) -class JavascriptUdf(Model, AuditMixin): - __tablename__ = 'udfs' +class Metric(Model): + __tablename__ = 'metrics' id = Column(Integer, primary_key=True) + metric_name = Column(String(512)) + verbose_name = Column(String(1024)) + metric_type = Column(String(32)) datasource_name = Column( String(256), ForeignKey('datasources.datasource_name')) - udf_name = Column(String(256)) - column_list = Column(String(1024)) - code = Column(Text) - - def __repr__(self): - return self.udf_name + datasource = relationship('Datasource', backref='metrics') + json = Column(Text) + @property + def json_obj(self): + return json.loads(self.json) class Column(Model, AuditMixin): __tablename__ = 'columns' @@ -107,6 +120,7 @@ class Column(Model, AuditMixin): datasource_name = Column( String(256), ForeignKey('datasources.datasource_name')) + datasource = relationship('Datasource', backref='columns') column_name = Column(String(256)) is_active = Column(Boolean, default=True) type = Column(String(32)) @@ -120,3 +134,73 @@ class Column(Model, AuditMixin): def __repr__(self): return self.column_name + @property + def isnum(self): + return self.type in ('LONG', 'DOUBLE') + + def generate_metrics(self): + M = Metric + metrics = [] + metrics.append(Metric( + metric_name='count', + verbose_name='COUNT(*)', + metric_type='count', + json=json.dumps({ + 'type': 'count', 'name': 'count'}) + )) + if self.datasource.datasource_name == 'platform' and self.column_name=='subject_id': + print((self.column_name, self.type, self.isnum)) + + if self.sum and self.isnum: + mt = self.type.lower() + 'Sum' + name='sum__' + self.column_name + metrics.append(Metric( + metric_name=name, + metric_type='sum', + verbose_name='SUM({})'.format(self.column_name), + json=json.dumps({ + 'type': mt, 'name': name, 'fieldName': self.column_name}) + )) + if self.min and self.isnum: + mt = self.type.lower() + 'Min' + name='min__' + self.column_name + metrics.append(Metric( + metric_name=name, + metric_type='min', + verbose_name='MIN({})'.format(self.column_name), + json=json.dumps({ + 'type': mt, 'name': name, 'fieldName': self.column_name}) + )) + if self.max and self.isnum: + mt = self.type.lower() + 'Max' + name='max__' + self.column_name + metrics.append(Metric( + metric_name=name, + metric_type='max', + verbose_name='MAX({})'.format(self.column_name), + json=json.dumps({ + 'type': mt, 'name': name, 'fieldName': self.column_name}) + )) + if self.count_distinct: + mt = 'count_distinct' + name='count_distinct__' + self.column_name + metrics.append(Metric( + metric_name=name, + verbose_name='COUNT(DISTINCT {})'.format(self.column_name), + metric_type='count_distinct', + json=json.dumps({ + 'type': 'cardinality', + 'name': name, + 'fieldNames': [self.column_name]}) + )) + for metric in metrics: + m = ( + db.session.query(M) + .filter(M.datasource_name==self.datasource_name) + .filter(M.metric_name==metric.metric_name) + .first() + ) + metric.datasource_name = self.datasource_name + if not m: + db.session.add(metric) + db.session.commit() diff --git a/panoramix/app/static/chaudron_white.png b/panoramix/app/static/chaudron_white.png new file mode 100644 index 0000000000..8e634c8b28 Binary files /dev/null and b/panoramix/app/static/chaudron_white.png differ diff --git a/panoramix/app/templates/appbuilder/navbar.html b/panoramix/app/templates/appbuilder/navbar.html new file mode 100644 index 0000000000..ad1befa036 --- /dev/null +++ b/panoramix/app/templates/appbuilder/navbar.html @@ -0,0 +1,35 @@ + + + +{% set menu = appbuilder.menu %} +{% set languages = appbuilder.languages %} + + diff --git a/panoramix/app/templates/panoramix/base.html b/panoramix/app/templates/panoramix/base.html index b93e5d1f05..56818d2dd8 100644 --- a/panoramix/app/templates/panoramix/base.html +++ b/panoramix/app/templates/panoramix/base.html @@ -4,5 +4,8 @@ {{super()}} {% endblock %} diff --git a/panoramix/app/views.py b/panoramix/app/views.py index 1f4fb07752..90a3eaa80c 100644 --- a/panoramix/app/views.py +++ b/panoramix/app/views.py @@ -38,8 +38,7 @@ def form_factory(datasource, form_args=None): viz_type = SelectField( 'Viz', choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()]) - metric = SelectField( - 'Metric', choices=[(m, m) for m in datasource.metrics]) + metric = SelectField('Metric', choices=datasource.metrics_combo) groupby = SelectMultipleField( 'Group by', choices=[ (s, s) for s in datasource.groupby_column_names]) @@ -61,22 +60,33 @@ def form_factory(datasource, form_args=None): class ColumnInlineView(CompactCRUDMixin, ModelView): datamodel = SQLAInterface(models.Column) - edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max'] - list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max'] + edit_columns = [ + 'column_name', 'datasource', 'groupby', 'count_distinct', + 'sum', 'min', 'max'] + list_columns = [ + 'column_name', 'type', 'groupby', 'count_distinct', + 'sum', 'min', 'max'] can_delete = False appbuilder.add_view_no_menu(ColumnInlineView) -class JavascriptUdfInlineView(CompactCRUDMixin, ModelView): - datamodel = SQLAInterface(models.JavascriptUdf) - edit_columns = ['udf_name', 'column_list', 'code'] -appbuilder.add_view_no_menu(JavascriptUdfInlineView) + +class MetricInlineView(CompactCRUDMixin, ModelView): + datamodel = SQLAInterface(models.Metric) + list_columns = ['metric_name', 'verbose_name', 'metric_type' ] + edit_columns = [ + 'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json'] + add_columns = [ + 'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json'] +appbuilder.add_view_no_menu(MetricInlineView) class DatasourceModelView(ModelView): datamodel = SQLAInterface(models.Datasource) - list_columns = ['datasource_link', 'is_featured' ] - related_views = [ColumnInlineView, JavascriptUdfInlineView] - edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden'] + list_columns = ['datasource_link', 'is_featured', 'is_hidden'] + related_views = [ColumnInlineView, MetricInlineView] + edit_columns = [ + 'datasource_name', 'description', 'is_featured', 'is_hidden', + 'default_endpoint'] page_size = 100 @@ -90,13 +100,18 @@ appbuilder.add_view( class Panoramix(BaseView): @expose("/datasource//") def datasource(self, datasource_name): - viz_type = request.args.get("viz_type", "table") + viz_type = request.args.get("viz_type") + datasource = ( db.session .query(models.Datasource) .filter_by(datasource_name=datasource_name) .first() ) + if not viz_type and datasource.default_endpoint: + return redirect(datasource.default_endpoint) + if not viz_type: + viz_type = "table" obj = viz.viz_types[viz_type]( datasource, form_class=form_factory(datasource, request.args), @@ -107,7 +122,7 @@ class Panoramix(BaseView): @expose("/refresh_datasources/") - def datasources(self): + def refresh_datasources(self): import requests import json endpoint = ( @@ -126,4 +141,6 @@ appbuilder.add_link( href='/panoramix/refresh_datasources/', category='Admin', icon="fa-cogs") + +#models.Metric.__table__.drop(db.engine) db.create_all() diff --git a/panoramix/app/viz.py b/panoramix/app/viz.py index c3f846e6ef..f1957aa6ee 100644 --- a/panoramix/app/viz.py +++ b/panoramix/app/viz.py @@ -23,7 +23,7 @@ class BaseViz(object): self.datasource = datasource self.form_class = form_class self.form_data = form_data - self.metric = form_data.get('metric') + self.metric = form_data.get('metric', 'count') self.df = self.bake_query() self.view = view if self.df is not None: @@ -67,7 +67,10 @@ class BaseViz(object): args = self.form_data groupby = args.getlist("groupby") or [] granularity = args.get("granularity") - metric = "count" + aggregations = { + m.metric_name: m.json_obj + for m in ds.metrics if m.metric_name == self.metric + } limit = int( args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT since = args.get("since", "all") @@ -77,12 +80,12 @@ class BaseViz(object): 'granularity': granularity or 'all', 'intervals': from_dttm + '/' + datetime.now().isoformat(), 'dimensions': groupby, - 'aggregations': {"count": agg.doublesum(metric)}, + 'aggregations': aggregations, 'limit_spec': { "type": "default", "limit": limit, "columns": [{ - "dimension": metric, + "dimension": self.metric, "direction": "descending", }], }, @@ -151,9 +154,9 @@ class TimeSeriesViz(HighchartsViz): columns=[ col for col in df.columns if col not in ["timestamp", metric]], values=[metric]) + chart_js = serialize( df, kind=self.chart_kind, stacked=self.stacked, **CHART_ARGS) - print self.stacked return super(TimeSeriesViz, self).render(chart_js=chart_js) def bake_query(self): diff --git a/panoramix/config.py b/panoramix/config.py index 0e43f631f3..333efa26c4 100644 --- a/panoramix/config.py +++ b/panoramix/config.py @@ -34,7 +34,7 @@ CSRF_ENABLED = True APP_NAME = "Panoramix" # Uncomment to setup Setup an App icon -#APP_ICON = "static/img/logo.jpg" +APP_ICON = "/static/chaudron_white.png" #---------------------------------------------------- # AUTHENTICATION CONFIG