From 29b0bdd00acb7bb5d103d9d37eb1a5fdaa914bb9 Mon Sep 17 00:00:00 2001 From: Maxime Date: Thu, 16 Jul 2015 00:38:03 +0000 Subject: [PATCH] Stable --- panoramix/TODO.md | 4 +- panoramix/app/models.py | 124 +++++++++++++++--- panoramix/app/static/chaudron_white.png | Bin 0 -> 2292 bytes .../app/templates/appbuilder/navbar.html | 35 +++++ panoramix/app/templates/panoramix/base.html | 3 + panoramix/app/views.py | 43 ++++-- panoramix/app/viz.py | 13 +- panoramix/config.py | 2 +- 8 files changed, 183 insertions(+), 41 deletions(-) create mode 100644 panoramix/app/static/chaudron_white.png create mode 100644 panoramix/app/templates/appbuilder/navbar.html diff --git a/panoramix/TODO.md b/panoramix/TODO.md index 192021631f..cc31f88ea6 100644 --- a/panoramix/TODO.md +++ b/panoramix/TODO.md @@ -1,3 +1,3 @@ # TODO -* Default URL params per datasource -* Get config metrics to work +* Multi-filters +* multi-metrics diff --git a/panoramix/app/models.py b/panoramix/app/models.py index b9b81d29e6..39259323ac 100644 --- a/panoramix/app/models.py +++ b/panoramix/app/models.py @@ -5,14 +5,9 @@ from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean from sqlalchemy.orm import relationship from app import db, utils from dateutil.parser import parse -""" - -You can use the extra Flask-AppBuilder fields and Mixin's - -AuditMixin will add automatic timestamp of created and modified by who +import json -""" client = utils.get_pydruid_client() class Datasource(Model, AuditMixin): @@ -22,12 +17,11 @@ class Datasource(Model, AuditMixin): is_featured = Column(Boolean, default=False) is_hidden = Column(Boolean, default=False) description = Column(Text) - columns = relationship('Column', backref='datasource') - udfs = relationship('JavascriptUdf', backref='datasource') + default_endpoint = Column(Text) @property - def metrics(self): - return [col.column_name for col in self.columns if not col.groupby] + def metrics_combo(self): + return [(m.metric_name, m.verbose_name) for m in self.metrics] def __repr__(self): return self.datasource_name @@ -37,9 +31,18 @@ class Datasource(Model, AuditMixin): url = "/panoramix/datasource/{}/".format(self.datasource_name) return '{self.datasource_name}'.format(**locals()) + def get_metric_obj(self, metric_name): + return [ + m.json_obj for m in self.metrics + if m.metric_name == metric_name + ][0] + @classmethod def latest_metadata(cls, name): results = client.time_boundary(datasource=name) + print "---" * 100 + print name + print results max_time = results[0]['result']['maxTime'] max_time = parse(max_time) intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' @@ -47,7 +50,13 @@ class Datasource(Model, AuditMixin): segment_metadata = client.segment_metadata( datasource=name, intervals=intervals) - return segment_metadata[-1]['columns'] + print segment_metadata + if segment_metadata: + return segment_metadata[-1]['columns'] + + def generate_metrics(self): + for col in self.columns: + col.generate_metrics() @classmethod def sync_to_db(cls, name): @@ -55,6 +64,8 @@ class Datasource(Model, AuditMixin): if not datasource: db.session.add(cls(datasource_name=name)) cols = cls.latest_metadata(name) + if not cols: + return for col in cols: col_obj = ( db.session @@ -71,7 +82,7 @@ class Datasource(Model, AuditMixin): col_obj.filterable = True if col_obj: col_obj.type = cols[col]['type'] - + col_obj.generate_metrics() db.session.commit() @property @@ -87,19 +98,21 @@ class Datasource(Model, AuditMixin): return sorted([c.column_name for c in self.columns if c.filterable]) -class JavascriptUdf(Model, AuditMixin): - __tablename__ = 'udfs' +class Metric(Model): + __tablename__ = 'metrics' id = Column(Integer, primary_key=True) + metric_name = Column(String(512)) + verbose_name = Column(String(1024)) + metric_type = Column(String(32)) datasource_name = Column( String(256), ForeignKey('datasources.datasource_name')) - udf_name = Column(String(256)) - column_list = Column(String(1024)) - code = Column(Text) - - def __repr__(self): - return self.udf_name + datasource = relationship('Datasource', backref='metrics') + json = Column(Text) + @property + def json_obj(self): + return json.loads(self.json) class Column(Model, AuditMixin): __tablename__ = 'columns' @@ -107,6 +120,7 @@ class Column(Model, AuditMixin): datasource_name = Column( String(256), ForeignKey('datasources.datasource_name')) + datasource = relationship('Datasource', backref='columns') column_name = Column(String(256)) is_active = Column(Boolean, default=True) type = Column(String(32)) @@ -120,3 +134,73 @@ class Column(Model, AuditMixin): def __repr__(self): return self.column_name + @property + def isnum(self): + return self.type in ('LONG', 'DOUBLE') + + def generate_metrics(self): + M = Metric + metrics = [] + metrics.append(Metric( + metric_name='count', + verbose_name='COUNT(*)', + metric_type='count', + json=json.dumps({ + 'type': 'count', 'name': 'count'}) + )) + if self.datasource.datasource_name == 'platform' and self.column_name=='subject_id': + print((self.column_name, self.type, self.isnum)) + + if self.sum and self.isnum: + mt = self.type.lower() + 'Sum' + name='sum__' + self.column_name + metrics.append(Metric( + metric_name=name, + metric_type='sum', + verbose_name='SUM({})'.format(self.column_name), + json=json.dumps({ + 'type': mt, 'name': name, 'fieldName': self.column_name}) + )) + if self.min and self.isnum: + mt = self.type.lower() + 'Min' + name='min__' + self.column_name + metrics.append(Metric( + metric_name=name, + metric_type='min', + verbose_name='MIN({})'.format(self.column_name), + json=json.dumps({ + 'type': mt, 'name': name, 'fieldName': self.column_name}) + )) + if self.max and self.isnum: + mt = self.type.lower() + 'Max' + name='max__' + self.column_name + metrics.append(Metric( + metric_name=name, + metric_type='max', + verbose_name='MAX({})'.format(self.column_name), + json=json.dumps({ + 'type': mt, 'name': name, 'fieldName': self.column_name}) + )) + if self.count_distinct: + mt = 'count_distinct' + name='count_distinct__' + self.column_name + metrics.append(Metric( + metric_name=name, + verbose_name='COUNT(DISTINCT {})'.format(self.column_name), + metric_type='count_distinct', + json=json.dumps({ + 'type': 'cardinality', + 'name': name, + 'fieldNames': [self.column_name]}) + )) + for metric in metrics: + m = ( + db.session.query(M) + .filter(M.datasource_name==self.datasource_name) + .filter(M.metric_name==metric.metric_name) + .first() + ) + metric.datasource_name = self.datasource_name + if not m: + db.session.add(metric) + db.session.commit() diff --git a/panoramix/app/static/chaudron_white.png b/panoramix/app/static/chaudron_white.png new file mode 100644 index 0000000000000000000000000000000000000000..8e634c8b2814ae1f208e235f5fc071b8fe7a3081 GIT binary patch literal 2292 zcmV1h2g4nnlz}*1u25eSv zmk5IEv}75bJ!Qt)&*vCY(#P;;U&}P@-$vM={6|Zc(b02X)~he)vP`@Axm*2WJO8%C zgX3{GJlDz3X2Y&pFH3hjYRP=X_G&@nNh@aMZi-7YPmVmgZK(enLWY!e7v_6hE}pw7 zF6D0b7fRJ z;0P|2yE!_+MBUtgyJ5u+S5K9%yS;O_@;ZLdp{um}IfA?S7#7<3$4~|EU2KikS){!c zm*W2ep|yazYE;49knO;2!{B}Yp>*h+<8_2?K$c6|0^y< zX=3YM48>M+J4Yv=I3n#kY*zSCI648vxmy!KY^&^^;ch5Va4~KGO}_syhP$b(M=8mY z)iZrRY${M(iX!H3ZxIDIBodqAQV?9KLG0)R5L}{+A9t&BUej)6M}Opya!l~PMJ{1r z{tMa?Twa1xTw1T#HY2!5S2!&mQaU=JtU=YxHS`bZ@opn+)0dDm;cf*wSBU>FVX?Ql zBeu=dM#V@%+;HJ;n44^0w_J=EX~)K*Z5GMh6qjZmI$a2E{k5%7TpHtGAjPGL2i9-H zlIs>aDdHUNeWCYrbV9*{>-@x*v8%jIu8{;cEbwlQPAGhE9iGg-{i$)8Ae=0gb2!P} zw){m8i^PWvf7AYnztcp@{4KH^5X0TVhmyOkI51#B~8B11NvOoal&u=SJEd8+CDtz*xR_4`T#jPp$@@4dn#2|$nWdN^emE);BMuM zt*yPo#(0+y{zhy~CVj=-DoB(Z3jCc-6qgn-NG~;Y+T_j)B}mRorr8~xu&oT{wYJ65uncmI5Izn|huDrz z*r4cczOZd&zikRL`$t{x=BAuus(0jW+Z0{1mA!X==dw3Xv*No{xgh}$&VRH*@^D^p zDUe8Tj!po$8^GPT8?d6gDJ}&b6rAExAd%P{od9q*fV*)wU}bmfv?MSuHpQhtDjb~v za5sRvaW`OHcT-#nJm_wUOMyi0=I8_^@-9JC|=0001BD--G1_mcs+@Saua zP^8`BK<*4Vh^^BP*bv>?g1hPiY>4ZcVmoMmY>@q0f@`%sHb_fwzy<`@XnSms!LxUzhv=TcKZDaItPn0kU_uF^97=r@36D3Sb~truF7#mlK~abR56-RC+u$ zJ;xmY006K-Mtd)jANAhaGpd_Io($X#;Bt1Ppj!Hx#AY`NxLiWPo%AOPpsPY~wtb-x zt~I!uB2|<`Q@Y>=eTae>N>7#mk|&7-=gdk-o+K2UGb>Rd7duOUwCqb0oV7QU%t<&2 z2rGq?5LU`g0PGTN|=y4mL@I6F*GGp!va zKsq{t1JcpUPy*=~2o6X@c1r{94*!GYjx>jDT4AUJ{p zBuxNgI`CaItPf#nl9(8VM0M(K6-2#K|L4}-i6`aHgBurfB0ldV1mn+VvL}greIWxt( z`Un-#=Po!03ZW*NUG4}gQ3KOB-j#r`62eJgCGcAgyHZdehr+|jw_Qn$7Qswl!MYqW=d!Db@vBx& z0tGoTjmyKXHY8AxDW_en2rVI$v^IzBhLZbT?TikMg{)+9(YcEN=52{ZH|Te)z_8mY zqtlv^kmBmx7D}QJ9a3=JM3RK)Kx{+@Vk0`D1F;buh^>0j4aN&zV4$Sm3(tYFc!r*O zo)_J8+@8Icc!X)39I57Y(KY6EcwoNPP>XpTmgg)vvYhaaLQDuD)4Scpc-u4=uC{{P zIf8Fng#K=g{&n3>ZQVxF{ah`bunzaWZsXUfV3I0zGWF|WCdwhin0z~Y=w1g9-Zq4G zxShvuzRoSOyPf_cL)Fi8NN;C{2ij`IJm`l<-XRFI z4;fM+=uC(h+Nr>RPdnSmfp72Jo#RS;JnkIx@`5Hgz&qE&<^eRc(EksB^Kqwz&qWFvDDCXdhZ~rOZYvfSA?~e%LQz1o*B4HhHk4|-Vnmm zHauMz#QooJ_4XO#*Zp08_&@ra=XmpOYqd>veO}^jk|pT0axohlqK3d)`QMGc^af<> zGxe;uK7(B-2`A6e#2Z=$vI}KChF|*YCR-&tEy@Xvd`|FXGijDIvkPTRa}%oA-CW-E zB0)O%pYOfWV&Ee!ww00026GXDb==(;@q!9bz_ O0000 +
+
+ + {% if appbuilder.app_icon %} + + + + {% endif %} + + + {{ appbuilder.app_name }} + + +
+
+
    + {% include 'appbuilder/navbar_menu.html' %} +
+
    + {% include 'appbuilder/navbar_right.html' %} +
+
+
+ diff --git a/panoramix/app/templates/panoramix/base.html b/panoramix/app/templates/panoramix/base.html index b93e5d1f05..56818d2dd8 100644 --- a/panoramix/app/templates/panoramix/base.html +++ b/panoramix/app/templates/panoramix/base.html @@ -4,5 +4,8 @@ {{super()}} {% endblock %} diff --git a/panoramix/app/views.py b/panoramix/app/views.py index 1f4fb07752..90a3eaa80c 100644 --- a/panoramix/app/views.py +++ b/panoramix/app/views.py @@ -38,8 +38,7 @@ def form_factory(datasource, form_args=None): viz_type = SelectField( 'Viz', choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()]) - metric = SelectField( - 'Metric', choices=[(m, m) for m in datasource.metrics]) + metric = SelectField('Metric', choices=datasource.metrics_combo) groupby = SelectMultipleField( 'Group by', choices=[ (s, s) for s in datasource.groupby_column_names]) @@ -61,22 +60,33 @@ def form_factory(datasource, form_args=None): class ColumnInlineView(CompactCRUDMixin, ModelView): datamodel = SQLAInterface(models.Column) - edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max'] - list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max'] + edit_columns = [ + 'column_name', 'datasource', 'groupby', 'count_distinct', + 'sum', 'min', 'max'] + list_columns = [ + 'column_name', 'type', 'groupby', 'count_distinct', + 'sum', 'min', 'max'] can_delete = False appbuilder.add_view_no_menu(ColumnInlineView) -class JavascriptUdfInlineView(CompactCRUDMixin, ModelView): - datamodel = SQLAInterface(models.JavascriptUdf) - edit_columns = ['udf_name', 'column_list', 'code'] -appbuilder.add_view_no_menu(JavascriptUdfInlineView) + +class MetricInlineView(CompactCRUDMixin, ModelView): + datamodel = SQLAInterface(models.Metric) + list_columns = ['metric_name', 'verbose_name', 'metric_type' ] + edit_columns = [ + 'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json'] + add_columns = [ + 'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json'] +appbuilder.add_view_no_menu(MetricInlineView) class DatasourceModelView(ModelView): datamodel = SQLAInterface(models.Datasource) - list_columns = ['datasource_link', 'is_featured' ] - related_views = [ColumnInlineView, JavascriptUdfInlineView] - edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden'] + list_columns = ['datasource_link', 'is_featured', 'is_hidden'] + related_views = [ColumnInlineView, MetricInlineView] + edit_columns = [ + 'datasource_name', 'description', 'is_featured', 'is_hidden', + 'default_endpoint'] page_size = 100 @@ -90,13 +100,18 @@ appbuilder.add_view( class Panoramix(BaseView): @expose("/datasource//") def datasource(self, datasource_name): - viz_type = request.args.get("viz_type", "table") + viz_type = request.args.get("viz_type") + datasource = ( db.session .query(models.Datasource) .filter_by(datasource_name=datasource_name) .first() ) + if not viz_type and datasource.default_endpoint: + return redirect(datasource.default_endpoint) + if not viz_type: + viz_type = "table" obj = viz.viz_types[viz_type]( datasource, form_class=form_factory(datasource, request.args), @@ -107,7 +122,7 @@ class Panoramix(BaseView): @expose("/refresh_datasources/") - def datasources(self): + def refresh_datasources(self): import requests import json endpoint = ( @@ -126,4 +141,6 @@ appbuilder.add_link( href='/panoramix/refresh_datasources/', category='Admin', icon="fa-cogs") + +#models.Metric.__table__.drop(db.engine) db.create_all() diff --git a/panoramix/app/viz.py b/panoramix/app/viz.py index c3f846e6ef..f1957aa6ee 100644 --- a/panoramix/app/viz.py +++ b/panoramix/app/viz.py @@ -23,7 +23,7 @@ class BaseViz(object): self.datasource = datasource self.form_class = form_class self.form_data = form_data - self.metric = form_data.get('metric') + self.metric = form_data.get('metric', 'count') self.df = self.bake_query() self.view = view if self.df is not None: @@ -67,7 +67,10 @@ class BaseViz(object): args = self.form_data groupby = args.getlist("groupby") or [] granularity = args.get("granularity") - metric = "count" + aggregations = { + m.metric_name: m.json_obj + for m in ds.metrics if m.metric_name == self.metric + } limit = int( args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT since = args.get("since", "all") @@ -77,12 +80,12 @@ class BaseViz(object): 'granularity': granularity or 'all', 'intervals': from_dttm + '/' + datetime.now().isoformat(), 'dimensions': groupby, - 'aggregations': {"count": agg.doublesum(metric)}, + 'aggregations': aggregations, 'limit_spec': { "type": "default", "limit": limit, "columns": [{ - "dimension": metric, + "dimension": self.metric, "direction": "descending", }], }, @@ -151,9 +154,9 @@ class TimeSeriesViz(HighchartsViz): columns=[ col for col in df.columns if col not in ["timestamp", metric]], values=[metric]) + chart_js = serialize( df, kind=self.chart_kind, stacked=self.stacked, **CHART_ARGS) - print self.stacked return super(TimeSeriesViz, self).render(chart_js=chart_js) def bake_query(self): diff --git a/panoramix/config.py b/panoramix/config.py index 0e43f631f3..333efa26c4 100644 --- a/panoramix/config.py +++ b/panoramix/config.py @@ -34,7 +34,7 @@ CSRF_ENABLED = True APP_NAME = "Panoramix" # Uncomment to setup Setup an App icon -#APP_ICON = "static/img/logo.jpg" +APP_ICON = "/static/chaudron_white.png" #---------------------------------------------------- # AUTHENTICATION CONFIG