Numerous improvements

2015-07-20 23:29:16 +00:00 · 2015-07-20 23:29:16 +00:00 · 4334373a33
parent d268b6b231
commit 4334373a33
11 changed files with 234 additions and 194 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -0,0 +1,93 @@
+# Contributing
+
+Contributions are welcome and are greatly appreciated! Every
+little bit helps, and credit will always be given.
+
+You can contribute in many ways:
+
+## Types of Contributions
+
+### Report Bugs
+
+Report bugs through Gihub
+
+If you are reporting a bug, please include:
+
+-   Your operating system name and version.
+-   Any details about your local setup that might be helpful in
+    troubleshooting.
+-   Detailed steps to reproduce the bug.
+
+### Fix Bugs
+
+Look through the GitHub issues for bugs. Anything tagged with "bug" is
+open to whoever wants to implement it.
+
+### Implement Features
+
+Look through the GitHub issues for features. Anything tagged with
+"feature" is open to whoever wants to implement it.
+
+We've created the operators, hooks, macros and executors we needed, but we 
+made sure that this part of Airflow is extensible. New operators,
+hooks and operators are very welcomed!
+
+### Documentation
+
+Airflow could always use better documentation,
+whether as part of the official Airflow docs,
+in docstrings, `docs/*.rst` or even on the web as blog posts or
+articles.
+
+### Submit Feedback
+
+The best way to send feedback is to file an issue on Github.
+
+If you are proposing a feature:
+
+-   Explain in detail how it would work.
+-   Keep the scope as narrow as possible, to make it easier to
+    implement.
+-   Remember that this is a volunteer-driven project, and that
+    contributions are welcome :)
+
+## Latests Documentation
+
+[API Documentation](http://pythonhosted.com/airflow)
+
+## Testing
+
+Install development requirements:
+
+    pip install -r requirements.txt
+
+Tests can then be run with:
+
+    ./run_unit_tests.sh
+
+Lint the project with:
+
+    flake8 changes tests
+
+## API documentation
+
+Generate the documentation with:
+
+    cd docs && ./build.sh
+
+
+## Pull Request Guidelines
+
+Before you submit a pull request from your forked repo, check that it 
+meets these guidelines:
+
+1.  The pull request should include tests, either as doctests,
+    unit tests, or both.
+2.  If the pull request adds functionality, the docs should be updated
+    as part of the same PR. Doc string are often sufficient, make 
+    sure to follow the sphinx compatible standards.
+3.  The pull request should work for Python 2.6, 2.7, and ideally python 3.3.
+    `from __future__ import ` will be required in every `.py` file soon.
+4.  Code will be reviewed by re running the unittests, flake8 and syntax
+    should be as rigorous as the core Python project.
+5.  Please rebase and resolve all conflicts before submitting.
--- a/TODO.md
+++ b/TODO.md
@ -1,2 +1,8 @@
 # TODO
-* Multi-filters
+* STOCK CHART + compare time ranges
+* Save a chart
+* Datasource + Owner 
+* Column description
+* Label
+* CSV
+* Bookmarks / url shortener
--- a/app.db
+++ b/app.db
--- a/app/templates/appbuilder/navbar.html
+++ b/app/templates/appbuilder/navbar.html
@ -5,7 +5,7 @@
 {% set languages = appbuilder.languages %}

 <div class="navbar {{menu.extra_classes}}" role="navigation">
-   <div class="container">
+   <div class="container-fluid">
        <div class="navbar-header">
            <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
                <span class="icon-bar"></span>
--- a/app/templates/panoramix/datasource.html
+++ b/app/templates/panoramix/datasource.html
@ -6,10 +6,17 @@
  height: 70px; 
  overflow: auto;
 }
+.no-gutter > [class*='col-'] {
+    padding-right:0;
+    padding-left:0;
+}
+form div.select2-container.form-control {
+  margin-bottom: 5px;
+}
 </style>
 {% endblock %}
 {% block content %}
-<div class="container">
+<div class="container-fluid">
  <div class="col-md-3">
    <h3>
      {{ datasource.datasource_name }}
@ -17,7 +24,7 @@
    </h3>

    <hr>
-    <form method="GET">
+    <form id="query" method="GET" style="display: none;">
      <div>{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control select2") }}</div>
      <div>{{ form.metrics.label }}: {{ form.metrics(class_="form-control select2") }}</div>
      <div>{{ form.granularity.label }}: {{ form.granularity(class_="form-control select2_free_granularity") }}</div>
@ -31,26 +38,23 @@
      <div>{{ form.limit.label }}: {{ form.limit(class_="form-control select2") }}</div>
      <hr>
      <h4>Filters</h4>
-      <div id="filters">
-        {% for i in range(10) %}
-          <div id="flt{{ i }}" class="{{ "hidden" if i != 1 }}">
-              <span class="" style="width: 100px;">{{ form['flt_col_' ~ i](class_="form-control select2 inc") }}</span>
-            <div class="row">
-              <span class="col col-md-3">{{ form['flt_op_' ~ i](class_="form-control select2 input-sm inc") }}</span>
-              <span class="col col-md-7">{{ form['flt_eq_' ~ i](class_="form-control inc") }}</span>
-              <button type="col col-md-2" class="btn btn-sm" aria-label="Delete filter">
-                <span class="glyphicon glyphicon-minus" aria-hidden="true"></span>
-              </button>
-            </div>
-            <hr/>
+        <div id="flt0" style="display: none;">
+            <span class="">{{ form.flt_col_0(class_="form-control inc") }}</span>
+          <div class="row">
+            <span class="col col-sm-4">{{ form.flt_op_0(class_="form-control inc") }}</span>
+            <span class="col col-sm-6">{{ form.flt_eq_0(class_="form-control inc") }}</span>
+            <button type="button" class="btn btn-sm remove" aria-label="Delete filter">
+              <span class="glyphicon glyphicon-minus"  aria-hidden="true"></span>
+            </button>
          </div>
-        {% endfor %}
-      </div>
+          <hr style="margin: 5px 0px;"/>
+        </div>
+      <div id="filters"></div>
      <button type="button" id="plus" class="btn btn-sm" aria-label="Add a filter">
        <span class="glyphicon glyphicon-plus" aria-hidden="true"></span>
      </button>
      <hr>
-      <input type="submit" class="btn btn-primary" value="Druidify!">
+      <button type="button" class="btn btn-primary" id="druidify">Druidify!</button>
      <hr style="margin-bottom: 0px;">
      <img src="{{ url_for("static", filename="panoramix.png") }}" width=250>
    </form><br>
@ -81,7 +85,68 @@
 {{ super() }}
 <script>
 $( document ).ready(function() {
+  function getParam(name) {
+      name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]");
+      var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"),
+          results = regex.exec(location.search);
+      return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
+  }
+
  $(".select2").select2();
+  $("form").slideDown("slow");
+
+  function set_filters(){
+    for (var i=1; i<10; i++){
+      var eq = getParam("flt_eq_" + i);
+      if (eq !=''){
+        add_filter(i);
+      }
+    }
+  }
+  set_filters();
+
+  function add_filter(i) {
+      cp = $("#flt0").clone();
+      $(cp).appendTo("#filters");  
+      $(cp).slideDown("slow");
+      if (i != undefined){
+        $(cp).find("#flt_eq_0").val(getParam("flt_eq_" + i));
+        $(cp).find("#flt_op_0").val(getParam("flt_op_" + i));
+        $(cp).find("#flt_col_0").val(getParam("flt_col_" + i));
+      }
+
+      $(cp).find('select').select2();
+      $(cp).find('.remove').click(function() {
+        $(this).parent().parent().slideUp("slow", function(){$(this).remove()});
+      });
+  }
+  $("#plus").click(add_filter);
+  add_filter();
+  $("#druidify").click(function () {
+    var i = 1;
+
+    // removing empty filters
+    $("#filters > div").each(function(){
+      if ($(this).find("#flt_eq_0").val() == '')
+        $(this).slideUp();
+    });
+
+    // Assigning the right id to form elements in filters
+    $("#filters > div").each(function(){
+      $(this).attr("id", function(){return "flt_" + i;})
+      $(this).find("#flt_col_0")
+        .attr("id", function(){return "flt_col_" + i;})
+        .attr("name", function(){return "flt_col_" + i;});
+      $(this).find("#flt_op_0")
+        .attr("id", function(){return "flt_op_" + i;})
+        .attr("name", function(){return "flt_op_" + i;});
+      $(this).find("#flt_eq_0")
+        .attr("id", function(){return "flt_eq_" + i;})
+        .attr("name", function(){return "flt_eq_" + i;});
+      i++;
+    });
+    $("#query").submit();
+  });

  function create_choices (term, data) { 
      if ($(data).filter(function() { 
--- a/app/templates/panoramix/noadmin.html
+++ b/app/templates/panoramix/noadmin.html
@ -1,56 +0,0 @@
-{% extends "bootstrap/base.html" %}
-
-{% block title %}Panoramix - A Druid UI{% endblock %}
-{% block html_attribs %} lang="en"{% endblock %}
-
-{% block head %}
-{{super()}}
-<link rel="icon" type="image/png" href="{{url_for('.static', filename='chaudron.png')}}">
-{% endblock %}
-
-{% block styles %}
-{{super()}}
-<link rel="stylesheet" href="{{url_for('.static', filename='bootstrap-theme.css')}}">
-<link rel="stylesheet" href="{{url_for('.static', filename='main.css')}}">
-<link rel="stylesheet" href="{{url_for('.static', filename='select2.min.css')}}">
-<link rel="stylesheet" href="{{url_for('.static', filename='select2-bootstrap.css')}}">
-{% endblock %}
-
-{% block navbar %}
-<nav class="navbar navbar-inverse navbar-fixed-top">
-  <div class="container">
-    <!-- Brand and toggle get grouped for better mobile display -->
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <a class="navbar-brand" href="#" class="pull-left">
-          Panoramix
-        </a>
-    </div>
-
-    <!-- Collect the nav links, forms, and other content for toggling -->
-    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-      <ul class="nav navbar-nav">
-        <li class="active"><a href="#">Link <span class="sr-only">(current)</span></a></li>
-        <li class="dropdown">
-          <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Dropdown <span class="caret"></span></a>
-          <ul class="dropdown-menu">
-            <li><a href="#">Action</a></li>
-            <li><a href="#">Another action</a></li>
-            <li><a href="#">Something else here</a></li>
-            <li role="separator" class="divider"></li>
-            <li><a href="#">Separated link</a></li>
-            <li role="separator" class="divider"></li>
-            <li><a href="#">One more separated link</a></li>
-          </ul>
-        </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right"></ul>
-    </div><!-- /.navbar-collapse -->
-  </div><!-- /.container-fluid -->
-</nav>
-{% endblock %}
--- a/app/templates/panoramix/viz_highcharts.html
+++ b/app/templates/panoramix/viz_highcharts.html
@ -5,7 +5,11 @@

 {% block tail %}
 {{ super() }}
+{% if viz.chart_type == "stock" %}
+<script src="{{ url_for("static", filename="highstock.js") }}"></script>
+{% else %}
 <script src="{{ url_for("static", filename="highcharts.js") }}"></script>
+{% endif %}
 <script>
 $( document ).ready(function() {
    Highcharts.setOptions({
@ -13,6 +17,11 @@ $( document ).ready(function() {
            "#FF5A5F", "#007A87", "#7B0051", "#00D1C1", "#8CE071", "#FFB400",
            "#FFAA91", "#B4A76C", "#9CA299", "#565A5C"
        ],
+        global: {
+          useUTC: false
+        },
+  
+
    });
    $("#viz_type").click(function(){
        $("#queryform").submit();
--- a/app/views.py
+++ b/app/views.py
@ -1,7 +1,8 @@
 from datetime import timedelta
 import logging
+import json

-from flask import request, redirect, flash
+from flask import request, redirect, flash, Response
 from flask.ext.appbuilder.models.sqla.interface import SQLAInterface
 from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose
 from app import appbuilder, db, models, viz, utils
@ -59,7 +60,7 @@ def form_factory(datasource, form_args=None):
        setattr(QueryForm, 'flt_col_' + str(i), SelectField(
            'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names]))
        setattr(QueryForm, 'flt_op_' + str(i), SelectField(
-            'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]]))
+            'Filter 1', choices=[(m, m) for m in ['in', 'not in']]))
        setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super"))
    return QueryForm

@ -122,6 +123,11 @@ class Panoramix(BaseView):
            datasource,
            form_class=form_factory(datasource, request.args),
            form_data=request.args, view=self)
+        if request.args.get("json"):
+            return Response(
+                json.dumps(obj.get_query(), indent=4),
+                status=200,
+                mimetype="application/json")
        if obj.df is None or obj.df.empty:
            return obj.render_no_data()
        return obj.render()
@ -130,7 +136,6 @@ class Panoramix(BaseView):
    @expose("/refresh_datasources/")
    def refresh_datasources(self):
        import requests
-        import json
        endpoint = (
            "http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
            "{COORDINATOR_BASE_ENDPOINT}/datasources"
--- a/app/viz.py
+++ b/app/viz.py
@ -36,9 +36,8 @@ class BaseViz(object):
    def query_filters(self):
        args = self.form_data
        # Building filters
-        i = 1
        filters = None
-        while True:
+        for i in range(1, 10):
            col = args.get("flt_col_" + str(i))
            op = args.get("flt_op_" + str(i))
            eq = args.get("flt_eq_" + str(i))
@ -48,20 +47,25 @@ class BaseViz(object):
                    cond = Dimension(col)==eq
                elif op == '!=':
                    cond = ~(Dimension(col)==eq)
-                elif op == 'in':
+                elif op in ('in', 'not in'):
                    fields = []
-                    for s in eq.split(','):
-                        s = s.strip()
-                        fields.append(Filter.build_filter(Dimension(col)==s))
-                    cond = Filter(type="and", fields=fields)
-
+                    splitted = eq.split(',')
+                    if len(splitted) > 1:
+                        for s in eq.split(','):
+                            s = s.strip()
+                            fields.append(Filter.build_filter(Dimension(col)==s))
+                        cond = Filter(type="or", fields=fields)
+                    else:
+                        cond = Dimension(col)==eq
+                    if op == 'not in':
+                        cond = ~cond
                if filters:
-                    filters = cond and filters
+                    filters = Filter(type="and", fields=[
+                        Filter.build_filter(cond),
+                        Filter.build_filter(filters)
+                    ])
                else:
                    filters = cond
-            else:
-                break
-            i += 1
        return filters

    def query_obj(self):
@ -111,6 +115,11 @@ class BaseViz(object):
        client.groupby(**self.query_obj())
        return client.export_pandas()

+    def get_query(self):
+        client = utils.get_pydruid_client()
+        client.groupby(**self.query_obj())
+        return client.query_dict
+
    def df_prep(self, ):
        pass

@ -151,11 +160,14 @@ class HighchartsViz(BaseViz):
    template = 'panoramix/viz_highcharts.html'
    chart_kind = 'line'
    stacked = False
+    chart_type = 'not_stock'
+    compare = False


 class TimeSeriesViz(HighchartsViz):
    verbose_name = "Time Series - Line Chart"
-    chart_kind = "line"
+    chart_kind = "spline"
+    chart_type = 'stock'

    def render(self):
        metrics = self.metrics
@ -166,7 +178,10 @@ class TimeSeriesViz(HighchartsViz):
            values=metrics)

        chart_js = serialize(
-            df, kind=self.chart_kind, stacked=self.stacked, **CHART_ARGS)
+            df, kind=self.chart_kind,
+            viz=self,
+            compare=self.compare,
+            chart_type=self.chart_type, stacked=self.stacked, **CHART_ARGS)
        return super(TimeSeriesViz, self).render(chart_js=chart_js)

    def bake_query(self):
@ -199,6 +214,9 @@ class TimeSeriesViz(HighchartsViz):
            client.groupby(**qry)
        return client.export_pandas()

+class TimeSeriesCompareViz(TimeSeriesViz):
+    verbose_name = "Time Series - Percent Change"
+    compare = 'percent'

 class TimeSeriesAreaViz(TimeSeriesViz):
    verbose_name = "Time Series - Stacked Area Chart"
@ -259,9 +277,10 @@ class DistributionPieViz(HighchartsViz):
 viz_types = OrderedDict([
    ['table', TableViz],
    ['line', TimeSeriesViz],
+    ['compare', TimeSeriesCompareViz],
    ['area', TimeSeriesAreaViz],
    ['bar', TimeSeriesBarViz],
+    ['stacked_ts_bar', TimeSeriesStackedBarViz],
    ['dist_bar', DistributionBarViz],
    ['pie', DistributionPieViz],
-    ['stacked_ts_bar', TimeSeriesStackedBarViz],
 ])
--- a/templates/panoramix/base.html
+++ b/templates/panoramix/base.html
@ -1,62 +0,0 @@
-{% extends "bootstrap/base.html" %}
-
-{% block title %}Panoramix - A Druid UI{% endblock %}
-{% block html_attribs %} lang="en"{% endblock %}
-
-{% block styles %}
-{{super()}}
-<link rel="stylesheet" href="{{url_for('.static', filename='bootstrap-theme.css')}}">
-<link rel="stylesheet" href="{{url_for('.static', filename='main.css')}}">
-<link rel="stylesheet" href="{{url_for('.static', filename='select2.min.css')}}">
-<link rel="stylesheet" href="{{url_for('.static', filename='select2-bootstrap.css')}}">
-{% endblock %}
-
-{% block navbar %}
-<nav class="navbar navbar-inverse navbar-fixed-top">
-  <div class="container">
-    <!-- Brand and toggle get grouped for better mobile display -->
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <a class="navbar-brand" href="#">Panoramix</a>
-    </div>
-
-    <!-- Collect the nav links, forms, and other content for toggling -->
-    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-      <ul class="nav navbar-nav">
-        <li class="active"><a href="#">Link <span class="sr-only">(current)</span></a></li>
-        <li class="dropdown">
-          <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Dropdown <span class="caret"></span></a>
-          <ul class="dropdown-menu">
-            <li><a href="#">Action</a></li>
-            <li><a href="#">Another action</a></li>
-            <li><a href="#">Something else here</a></li>
-            <li role="separator" class="divider"></li>
-            <li><a href="#">Separated link</a></li>
-            <li role="separator" class="divider"></li>
-            <li><a href="#">One more separated link</a></li>
-          </ul>
-        </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-
-      </ul>
-    </div><!-- /.navbar-collapse -->
-  </div><!-- /.container-fluid -->
-</nav>
-{% endblock %}
-
-{% block scripts %}
-{{ super() }}
-<script src="{{url_for('.static', filename='select2.min.js')}}"></script>
-<script>
-$( document ).ready(function() {
-    $(".select2").select2();
-    $(".select2_tags").select2({tags: true});
-});
-</script>
-{% endblock %}
--- a/templates/panoramix/datasource.html
+++ b/templates/panoramix/datasource.html
@ -1,39 +0,0 @@
-{% extends "panoramix/base.html" %}
-{% block content %}
-<div class="container">
-    <div class="col-md-3">
-        <h3>{{ datasource }}</h3>
-        <form method="GET">
-            <div>{{ form.granularity.label }}: {{ form.granularity(class_="form-control select2") }}</div>
-            <div>{{ form.since.label }}: {{ form.since(class_="form-control select2") }}</div>
-            <div>{{ form.groupby.label }}: {{ form.groupby(class_="form-control select2") }}</div>
-            <div>{{ form.limit.label }}: {{ form.limit(class_="form-control select2_tags") }}</div>
-            <hr>
-            <h4>Filters</h4>
-            <div>
-                <span style="width: 100px;">{{ form.flt_col_1(class_="form-control select2") }}</span>
-                <span>{{ form.flt_op_1(class_="form-control select2 input-sm") }}</span>
-                <span>{{ form.flt_eq_1(class_="form-control") }}</span>
-            </div>
-            <hr>
-            <input type="submit" class="btn btn-primary">
-        </form><br>
-    </div>
-
-    <div class="col-md-9">
-
-        <h3>Tabular Data</h3>
-        {{ table|safe  }}
-
-        <h3>Results</h3>
-        <pre>
-           {{ results }} 
-        </pre>
-
-        <h3>Latest Segment Metadata</h3>
-        <pre>
-           {{ latest_metadata }} 
-        </pre>
-    </div>
-</div>
-{% endblock %}