Adding a foundation for unit tests

.gitignore

@@ -1,5 +1,6 @@
 *.pyc
 .DS_Store
+.coverage
 build
 *.db
 tmp

TODO.md

@@ -9,4 +9,18 @@
 * Create ~/.panoramix/ to host DB and config, generate default config there
 * Reintroduce query and stopwatch
 * Sort tooltip
-* Add a "Test Connection" button in Add Connection menu
+* Make "Test Connection" test further
+* Consistent colors for same entities
+* Contribution to total
+* Arbitrary expressions
+    * Group bucketing
+* ToT
+* Layers
+
+## Test
+* Line types
+* Intelligence around series name
+* Shapes
+* Line highlighting - draw attention
+
+## Bug

panoramix/__init__.py

@@ -6,13 +6,14 @@ from flask.ext.migrate import Migrate
 from panoramix import config
 
 APP_DIR = os.path.dirname(__file__)
+CONFIG_MODULE = os.environ.get('PANORAMIX_CONFIG', 'panoramix.config')
 
 # Logging configuration
 logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
 logging.getLogger().setLevel(logging.DEBUG)
 
 app = Flask(__name__)
-app.config.from_object('panoramix.config')
+app.config.from_object(CONFIG_MODULE)
 db = SQLA(app)
 migrate = Migrate(app, db, directory=APP_DIR + "/migrations")
 
@@ -23,7 +24,7 @@ class MyIndexView(IndexView):
 appbuilder = AppBuilder(
     app, db.session, base_template='panoramix/base.html',
     indexview=MyIndexView,
-    security_manager_class=config.CUSTOM_SECURITY_MANAGER)
+    security_manager_class=app.config.get("CUSTOM_SECURITY_MANAGER"))
 
 get_session = appbuilder.get_session
 from panoramix import views

panoramix/bin/panoramix

@@ -1,30 +1,35 @@
 #!/usr/bin/env python
 
-from flask.ext.script import Manager
-from panoramix import app, config
-from subprocess import Popen
-from flask.ext.migrate import MigrateCommand
-from panoramix import db
-from flask.ext.appbuilder import Base
-from sqlalchemy import Column, Integer, String
-from panoramix import config, models
 import csv
 import gzip
 import json
+from subprocess import Popen
 
+from flask.ext.script import Manager
+from flask.ext.migrate import MigrateCommand
+from panoramix import db
+from sqlalchemy import Column, Integer, String, Table
+
+from panoramix import app
+from panoramix import models
+
+
+config = app.config
 
 manager = Manager(app)
 manager.add_command('db', MigrateCommand)
 
+from flask.ext.appbuilder import Base
+
 @manager.option(
     '-d', '--debug', action='store_true',
     help="Start the web server in debug mode")
 @manager.option(
-    '-p', '--port', default=config.PANORAMIX_WEBSERVER_PORT,
+    '-p', '--port', default=config.get("PANORAMIX_WEBSERVER_PORT"),
     help="Specify the port on which to run the web server")
 def runserver(debug, port):
     """Starts a Panoramix web server"""
-    debug = debug or config.DEBUG
+    debug = debug or config.get("DEBUG")
     if debug:
         app.run(
             host='0.0.0.0',
@@ -45,22 +50,26 @@ def runserver(debug, port):
 def load_examples(sample):
     """Loads a set of Slices and Dashboards and a supporting dataset """
     print("Loading examples into {}".format(db))
-    class BirthNames(Base):
-        __tablename__ = "birth_names"
-        id = Column(Integer, primary_key=True)
-        state = Column(String(10))
-        year = Column(Integer)
-        name = Column(String(128))
-        num = Column(Integer)
-        ds = Column(String(20))
-        gender = Column(String(10))
+
+
+    BirthNames = Table(
+        "birth_names", Base.metadata,
+        Column("id", Integer, primary_key=True),
+        Column("state", String(10)),
+        Column("year", Integer),
+        Column("name", String(128)),
+        Column("num", Integer),
+        Column("ds", String(20)),
+        Column("gender", String(10)),
+    )
     try:
-        BirthNames.__table__.drop(db.engine)
+        BirthNames.drop(db.engine)
     except:
         pass
-    Base.metadata.create_all(db.engine)
+
+    BirthNames.create(db.engine)
     session = db.session()
-    with gzip.open(config.basedir + '/data/birth_names.csv.gz') as f:
+    with gzip.open(config.get("BASE_DIR") + '/data/birth_names.csv.gz') as f:
         bb_csv = csv.reader(f)
         for i, (state, year, name, gender, num) in enumerate(bb_csv):
             if i == 0:
@@ -68,24 +77,27 @@ def load_examples(sample):
             if num == "NA":
                 num = 0
             ds = str(year) + '-01-01'
-            session.add(
-                BirthNames(
-                    state=state, year=year,
-                    ds=ds,
-                    name=name, num=num, gender=gender))
-            if i % 1000 == 0:
+            db.engine.execute(
+                BirthNames.insert(),
+                state=state,
+                year=year,
+                ds=ds,
+                name=name, num=num, gender=gender)
+            if i % 5000 == 0:
                 print("{} loaded out of 82527 rows".format(i))
                 session.commit()
             session.commit()
             if sample and i>1000: break
     print("Done loading table!")
     print("-" * 80)
+
     print("Creating database reference")
     DB = models.Database
     dbobj = session.query(DB).filter_by(database_name='main').first()
     if not dbobj:
         dbobj = DB(database_name="main")
-    dbobj.sqlalchemy_uri = config.SQLALCHEMY_DATABASE_URI
+    print config.get("SQLALCHEMY_DATABASE_URI")
+    dbobj.sqlalchemy_uri = config.get("SQLALCHEMY_DATABASE_URI")
     session.add(dbobj)
     session.commit()
 
@@ -99,10 +111,10 @@ def load_examples(sample):
     obj.database = dbobj
     obj.columns = [models.TableColumn(
         column_name="num", sum=True, type="INTEGER")]
-    obj.fetch_metadata()
     models.Table
     session.add(obj)
     session.commit()
+    obj.fetch_metadata()
     tbl = obj
 
     print("Creating some slices")
@@ -121,7 +133,7 @@ def load_examples(sample):
             "groupby": [],
             "metric": 'sum__num',
             "metrics": ["sum__num"],
-            "row_limit": config.ROW_LIMIT,
+            "row_limit": config.get("ROW_LIMIT"),
             "since": "100 years",
             "slice_name": slice_name,
             "until": "now",
@@ -245,7 +257,7 @@ The source dataset came from [here](https://github.com/hadley/babynames)
             datasource_type='table',
             table=tbl,
             params=get_slice_json(
-                slice_name, viz_type="word_cloud", size_from="10", 
+                slice_name, viz_type="word_cloud", size_from="10",
                 groupby=['name'], size_to="70", rotation="square",
                 limit='100'))
         session.add(slc)

panoramix/config.py

@@ -2,7 +2,7 @@ import os
 from flask_appbuilder.security.manager import AUTH_DB
 # from flask_appbuilder.security.manager import (
 #    AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH)
-basedir = os.path.abspath(os.path.dirname(__file__))
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 from dateutil import tz
 
 """
@@ -107,10 +107,10 @@ LANGUAGES = {
 # Image and file configuration
 # ---------------------------------------------------
 # The file upload folder, when using models with files
-UPLOAD_FOLDER = basedir + '/app/static/uploads/'
+UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/'
 
 # The image upload folder, when using models with images
-IMG_UPLOAD_FOLDER = basedir + '/app/static/uploads/'
+IMG_UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/'
 
 # The image upload url, when using models with images
 IMG_UPLOAD_URL = '/static/uploads/'

panoramix/models.py

@@ -22,10 +22,12 @@ import sqlparse
 import requests
 import textwrap
 
-from panoramix import db, get_session, config, utils
+from panoramix import app, db, get_session, utils
 from panoramix.viz import viz_types
 from sqlalchemy.ext.declarative import declared_attr
 
+config = app.config
+
 QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration'])
 
 
@@ -402,6 +404,7 @@ class Table(Model, Queryable, AuditMixinNullable):
             df=df, duration=datetime.now() - qry_start_dttm, query=sql)
 
     def fetch_metadata(self):
+        table = self.database.get_table(self.table_name)
         try:
             table = self.database.get_table(self.table_name)
         except Exception as e:
@@ -673,8 +676,8 @@ class Datasource(Model, AuditMixin, Queryable):
         qry_start_dttm = datetime.now()
 
         # add tzinfo to native datetime with config
-        from_dttm = from_dttm.replace(tzinfo=config.DRUID_TZ)
-        to_dttm = to_dttm.replace(tzinfo=config.DRUID_TZ)
+        from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+        to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ"))
 
         query_str = ""
         aggregations = {

panoramix/templates/appbuilder/baselayout.html

@@ -2,8 +2,8 @@
 {% import 'appbuilder/baselib.html' as baselib %}
 
 {% block body %}
-        {% include 'appbuilder/general/confirm.html' %}
-        {% include 'appbuilder/general/alert.html' %}
+    {% include 'appbuilder/general/confirm.html' %}
+    {% include 'appbuilder/general/alert.html' %}
 
     {% block navbar %}
         <header class="top" role="header">
@@ -11,6 +11,8 @@
         </header>
     {% endblock %}
 
+    {% block uncontained %}{% endblock %}
+
     <div class="container">
       <div class="row">
           {% block messages %}
@@ -24,8 +26,6 @@
         {% block content_fluid %}
         {% endblock %}
     </div>
-    {% block uncontained %}
-    {% endblock %}
 
     {% block footer %}
         <footer>

panoramix/templates/index.html

@@ -127,7 +127,7 @@
     </div><!-- /.col-lg-4 -->
     <div class="col-lg-4">
       <img class="img-circle" src="{{ url_for('static', filename='gallery.jpg') }}" alt="Generic placeholder image" width="140" height="140">
-      <h2>Galery</h2>
+      <h2>Gallery</h2>
       <p>Navigate through the growing set of visualizations</p>
       <p><a class="btn btn-default" href="#" onclick="alert('Not ready yet!');" role="button">
         <i class="fa fa-picture-o"></i>

panoramix/templates/panoramix/models/database/macros.html

@@ -5,7 +5,7 @@
     $("#testconn").click(function() {
       var url = "/panoramix/testconn";
       $.ajax({
-        method: "GET",
+        method: "POST",
         url: url,
         data: { uri: $("#sqlalchemy_uri").val() }
       }).done(function() {

panoramix/views.py

@@ -11,7 +11,9 @@ from pydruid.client import doublesum
 from sqlalchemy import create_engine
 from wtforms.validators import ValidationError
 
-from panoramix import appbuilder, db, models, viz, utils, app, config
+from panoramix import appbuilder, db, models, viz, utils, app
+
+config = app.config
 
 
 def validate_json(form, field):
@@ -300,7 +302,7 @@ class Panoramix(BaseView):
             try:
                 resp = self.render_template("panoramix/viz.html", viz=obj)
             except Exception as e:
-                if config.DEBUG:
+                if config.get("DEBUG"):
                     raise(e)
                 return Response(
                     str(e),
@@ -326,12 +328,11 @@ class Panoramix(BaseView):
         return "SUCCESS"
 
     @has_access
-    @expose("/testconn/")
+    @expose("/testconn", methods=["POST"])
     def testconn(self):
         try:
-            db = create_engine(request.args.get('uri'))
-            for i in range(15):
-                request.args.get('uri')
+            uri = request.form.get('uri')
+            db = create_engine(uri)
             db.connect()
             return "SUCCESS"
         except Exception as e:

panoramix/viz.py

@@ -10,10 +10,12 @@ from werkzeug.urls import Href
 import numpy as np
 import pandas as pd
 
-from panoramix import utils, config
+from panoramix import app, utils
 from panoramix.highchart import Highchart, HighchartBubble
 from panoramix.forms import form_factory
 
+config = app.config
+
 CHART_ARGS = {
     'title': None,
 }
@@ -106,7 +108,7 @@ class BaseViz(object):
                 granularity).total_seconds() * 1000
         limit = int(args.get("limit", 0))
         row_limit = int(
-            args.get("row_limit", config.ROW_LIMIT))
+            args.get("row_limit", config.get("ROW_LIMIT")))
         since = args.get("since", "1 year ago")
         from_dttm = utils.parse_human_datetime(since)
         if from_dttm > datetime.now():

requirements.txt

@@ -1,6 +1,8 @@
+coverage
 flask
-flask-migrate
 flask-appbuilder
+flask-migrate
+flask-testing
 gunicorn
 markdown
 mysql-python