Cosmetics

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
+*.db
 tmp

app.py

@@ -1,132 +0,0 @@
-from pydruid import client
-from pydruid.utils.filters import Dimension
-from dateutil.parser import parse
-from datetime import datetime, timedelta
-from flask import Flask, render_template, request
-from flask_bootstrap import Bootstrap
-import json
-from wtforms import Form, SelectMultipleField, SelectField, TextField
-import pandas as pd
-pd.set_option('display.max_colwidth', -1)
-
-ROW_LIMIT = 10000
-PORT = 8088
-query = client.PyDruid("http://10.181.47.80:8080", 'druid/v2')
-
-app = Flask(__name__)
-Bootstrap(app)
-
-def latest_metadata(datasource):
-    max_time = query.time_boundary(datasource=datasource)[0]['result']['maxTime']
-    max_time = parse(max_time)
-    intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
-    intervals += max_time.isoformat()
-    return query.segment_metadata(
-        datasource=datasource,
-        intervals=intervals)[-1]['columns']
-
-@app.route("/datasource/<datasource>/")
-def datasource(datasource):
-
-    metadata = latest_metadata(datasource)
-    grain = ['all', 'none', 'minute', 'hour', 'day']
-    since_l = {
-        '1hour': timedelta(hours=1),
-        '1day': timedelta(days=1),
-        '7days': timedelta(days=7),
-        '28days': timedelta(days=28),
-        'all': timedelta(days=365*100)
-    }
-    limits = [0, 5, 10, 25, 50, 100, 500]
-    limit = request.args.get("limit")
-    try:
-        limit = int(limit)
-        if limit not in limits:
-            limits.append(limit)
-            limits = sorted(limits)
-    except:
-        pass
-    class QueryForm(Form):
-        groupby = SelectMultipleField(
-            'Group by', choices=[(m, m) for m in sorted(metadata.keys())])
-        granularity = SelectField(
-            'Granularity', choices=[(g, g) for g in grain])
-        since = SelectField(
-            'Since', choices=[(s, s) for s in since_l.keys()])
-        limit = SelectField(
-            'Limit', choices=[(s, s) for s in limits])
-        flt_col_1 = SelectField(
-            'Filter 1', choices=[(m, m) for m in sorted(metadata.keys())])
-        flt_op_1 = SelectField(
-            'Filter 1', choices=[(m, m) for m in ['==', 'in', '<', '>']])
-        flt_eq_1 = TextField("Super")
-
-    groupby = request.args.getlist("groupby") or []
-    granularity = request.args.get("granularity")
-    limit = int(request.args.get("limit", ROW_LIMIT)) or ROW_LIMIT
-    since = request.args.get("since", "all")
-    from_dttm = (datetime.now() - since_l[since]).isoformat()
-
-    # Building filters
-    i = 1
-    filters = []
-    while True:
-        col = request.args.get("flt_col_" + str(i))
-        op = request.args.get("flt_op_" + str(i))
-        eq = request.args.get("flt_eq_" + str(i))
-        print (col,op,eq)
-        if col and op and eq:
-            filters.append(Dimension(col)==eq)
-            filters = Dimension(col)==eq
-        else:
-            break
-        i += 1
-    print filters
-
-    results=[]
-    results = query.groupby(
-        datasource=datasource,
-        granularity=granularity or 'all',
-        intervals=from_dttm + '/' + datetime.now().isoformat(),
-        dimensions=groupby,
-        aggregations={"count": client.doublesum("count")},
-        filter=filters,
-        limit_spec={
-            "type": "default",
-            "limit": limit,
-            "columns": [{
-                "dimension" : "count",
-                "direction" : "descending",
-            },],
-        },
-    )
-
-    df = query.export_pandas()
-    if df is not None and not df.empty:
-        df = df.sort(df.columns[0], ascending=False)
-        if granularity == 'all':
-            del df['timestamp']
-
-        table = df.to_html(
-            classes=["table", "table-striped", 'table-bordered'], index=False)
-    else:
-        table = None
-
-    return render_template(
-        'panoramix/datasource.html',
-        table=table,
-        datasource=datasource,
-        latest_metadata=json.dumps(
-            metadata,
-            sort_keys=True,
-            indent=2),
-        results=json.dumps(
-            results,
-            sort_keys=True,
-            indent=2),
-        form=QueryForm(request.args),
-    )
-
-if __name__ == '__main__':
-    app.debug = True
-    app.run(host='0.0.0.0', port=PORT)

app/__init__.py

@@ -1,6 +1,6 @@
 import logging
 from flask import Flask
-from flask.ext.appbuilder import SQLA, AppBuilder
+from flask.ext.appbuilder import SQLA, AppBuilder, IndexView
 
 """
  Logging configuration
@@ -12,8 +12,13 @@ logging.getLogger().setLevel(logging.DEBUG)
 app = Flask(__name__)
 app.config.from_object('config')
 db = SQLA(app)
+
+class MyIndexView(IndexView):
+    index_template = 'index.html'
+
 appbuilder = AppBuilder(
-    app, db.session, base_template='panoramix/base.html')
+    app, db.session, base_template='panoramix/base.html',
+    indexview=MyIndexView)
 #appbuilder.app_name = 'Panoramix'
 
 
@@ -31,4 +36,3 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
 """
 
 from app import views
-

app/models.py

@@ -43,7 +43,7 @@ class Datasource(Model, AuditMixin):
         print "---" * 100
         print name
         print results
-        max_time = results[0]['result']['maxTime']
+        max_time = results[0]['result']['minTime']
         max_time = parse(max_time)
         intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
         intervals += (max_time + timedelta(seconds=1)).isoformat()

app/templates/index.html

@@ -1,2 +1,13 @@
-{% extends "appbuilder/baselayout.html" %}
+{% extends "appbuilder/base.html" %}
+{% block content %}
+<div class="jumbotron">
+  <div class="container">
+    <h1>Panoramix</h1>
+    <p>Panoramix is an interactive visualization platform built on top of Druid.io</p>
+  </div>
+</div>
+<div class="text-center">
+<img width="250" src="/static/tux_panoramix.png">
+</div>
+{% endblock %}
 

app/utils.py

@@ -1,5 +1,6 @@
 import config
-from datetime import timedelta
+from datetime import timedelta, datetime
+import parsedatetime
 
 since_l = {
     '1hour': timedelta(hours=1),
@@ -16,3 +17,15 @@ def get_pydruid_client():
         config.DRUID_BASE_ENDPOINT)
 
 
+def parse_human_datetime(s):
+    """
+    Use the parsedatetime lib to return ``datetime.datetime`` from human
+    generated strings
+
+    >>> parse_human_datetime("now") <= datetime.now()
+    True
+    """
+    cal = parsedatetime.Calendar()
+    d = cal.parse(s)[0]
+    return datetime(
+        d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec)

app/views.py

@@ -1,3 +1,6 @@
+from datetime import timedelta
+import logging
+
 from flask import request, redirect, flash
 from flask.ext.appbuilder.models.sqla.interface import SQLAInterface
 from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose
@@ -5,7 +8,6 @@ from app import appbuilder, db, models, viz, utils
 import config
 from wtforms import Form, SelectMultipleField, SelectField, TextField
 from wtforms.fields import Field
-from datetime import timedelta
 
 class OmgWtForm(Form):
     field_order = (
@@ -131,7 +133,11 @@ class Panoramix(BaseView):
         ).format(**config.__dict__)
         datasources = json.loads(requests.get(endpoint).text)
         for datasource in datasources:
-            models.Datasource.sync_to_db(datasource)
+            try:
+                models.Datasource.sync_to_db(datasource)
+            except Exception as e:
+                logging.exception(e)
+                logging.error("Failed at syncing " + datasource)
         flash("Refreshed metadata from Druid!", 'info')
         return redirect("/datasourcemodelview/list/")
 

app/viz.py

@@ -200,6 +200,7 @@ class TimeSeriesBarViz(TimeSeriesViz):
     verbose_name = "Time Series - Bar Chart"
     chart_kind = "bar"
 
+
 class TimeSeriesStackedBarViz(TimeSeriesViz):
     verbose_name = "Time Series - Stacked Bar Chart"
     chart_kind = "bar"

config.py

@@ -76,14 +76,17 @@ BABEL_DEFAULT_LOCALE = 'en'
 BABEL_DEFAULT_FOLDER = 'translations'
 # The allowed translation for you app
 LANGUAGES = {
-    'en': {'flag':'gb', 'name':'English'},
-    'pt': {'flag':'pt', 'name':'Portuguese'},
-    'pt_BR': {'flag':'br', 'name': 'Pt Brazil'},
-    'es': {'flag':'es', 'name':'Spanish'},
-    'de': {'flag':'de', 'name':'German'},
-    'zh': {'flag':'cn', 'name':'Chinese'},
-    'ru': {'flag':'ru', 'name':'Russian'}
+    'en': {'flag':'us', 'name':'English'},
+    'fr': {'flag':'fr', 'name':'French'},
 }
+"""
+'pt': {'flag':'pt', 'name':'Portuguese'},
+'pt_BR': {'flag':'br', 'name': 'Pt Brazil'},
+'es': {'flag':'es', 'name':'Spanish'},
+'de': {'flag':'de', 'name':'German'},
+'zh': {'flag':'cn', 'name':'Chinese'},
+'ru': {'flag':'ru', 'name':'Russian'}
+"""
 #---------------------------------------------------
 # Image and file configuration
 #---------------------------------------------------

requirements.txt

@@ -1,4 +1,5 @@
 pydruid
+parsedatetime
 python-dateutil
 flask
 flask-appbuilder