From c4b24cb9cca3797dbfa85fd3f0c15d7ffb99369b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 25 Sep 2015 15:43:50 -0700 Subject: [PATCH] Adding a foundation for unit tests --- .gitignore | 1 + TODO.md | 16 +++- panoramix/__init__.py | 5 +- panoramix/bin/panoramix | 76 +++++++++++-------- panoramix/config.py | 6 +- panoramix/models.py | 9 ++- .../templates/appbuilder/baselayout.html | 8 +- panoramix/templates/index.html | 2 +- .../panoramix/models/database/macros.html | 2 +- panoramix/views.py | 13 ++-- panoramix/viz.py | 6 +- requirements.txt | 4 +- 12 files changed, 92 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index fe3f9c11a7..d939c0dcb0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .DS_Store +.coverage build *.db tmp diff --git a/TODO.md b/TODO.md index 1aab72b268..db72db9fc1 100644 --- a/TODO.md +++ b/TODO.md @@ -9,4 +9,18 @@ * Create ~/.panoramix/ to host DB and config, generate default config there * Reintroduce query and stopwatch * Sort tooltip -* Add a "Test Connection" button in Add Connection menu +* Make "Test Connection" test further +* Consistent colors for same entities +* Contribution to total +* Arbitrary expressions + * Group bucketing +* ToT +* Layers + +## Test +* Line types +* Intelligence around series name +* Shapes +* Line highlighting - draw attention + +## Bug diff --git a/panoramix/__init__.py b/panoramix/__init__.py index 4879940788..e745047dc3 100644 --- a/panoramix/__init__.py +++ b/panoramix/__init__.py @@ -6,13 +6,14 @@ from flask.ext.migrate import Migrate from panoramix import config APP_DIR = os.path.dirname(__file__) +CONFIG_MODULE = os.environ.get('PANORAMIX_CONFIG', 'panoramix.config') # Logging configuration logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s') logging.getLogger().setLevel(logging.DEBUG) app = Flask(__name__) -app.config.from_object('panoramix.config') +app.config.from_object(CONFIG_MODULE) db = SQLA(app) migrate = Migrate(app, db, directory=APP_DIR + "/migrations") @@ -23,7 +24,7 @@ class MyIndexView(IndexView): appbuilder = AppBuilder( app, db.session, base_template='panoramix/base.html', indexview=MyIndexView, - security_manager_class=config.CUSTOM_SECURITY_MANAGER) + security_manager_class=app.config.get("CUSTOM_SECURITY_MANAGER")) get_session = appbuilder.get_session from panoramix import views diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index d51d940a35..565df33412 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -1,30 +1,35 @@ #!/usr/bin/env python -from flask.ext.script import Manager -from panoramix import app, config -from subprocess import Popen -from flask.ext.migrate import MigrateCommand -from panoramix import db -from flask.ext.appbuilder import Base -from sqlalchemy import Column, Integer, String -from panoramix import config, models import csv import gzip import json +from subprocess import Popen +from flask.ext.script import Manager +from flask.ext.migrate import MigrateCommand +from panoramix import db +from sqlalchemy import Column, Integer, String, Table + +from panoramix import app +from panoramix import models + + +config = app.config manager = Manager(app) manager.add_command('db', MigrateCommand) +from flask.ext.appbuilder import Base + @manager.option( '-d', '--debug', action='store_true', help="Start the web server in debug mode") @manager.option( - '-p', '--port', default=config.PANORAMIX_WEBSERVER_PORT, + '-p', '--port', default=config.get("PANORAMIX_WEBSERVER_PORT"), help="Specify the port on which to run the web server") def runserver(debug, port): """Starts a Panoramix web server""" - debug = debug or config.DEBUG + debug = debug or config.get("DEBUG") if debug: app.run( host='0.0.0.0', @@ -45,22 +50,26 @@ def runserver(debug, port): def load_examples(sample): """Loads a set of Slices and Dashboards and a supporting dataset """ print("Loading examples into {}".format(db)) - class BirthNames(Base): - __tablename__ = "birth_names" - id = Column(Integer, primary_key=True) - state = Column(String(10)) - year = Column(Integer) - name = Column(String(128)) - num = Column(Integer) - ds = Column(String(20)) - gender = Column(String(10)) + + + BirthNames = Table( + "birth_names", Base.metadata, + Column("id", Integer, primary_key=True), + Column("state", String(10)), + Column("year", Integer), + Column("name", String(128)), + Column("num", Integer), + Column("ds", String(20)), + Column("gender", String(10)), + ) try: - BirthNames.__table__.drop(db.engine) + BirthNames.drop(db.engine) except: pass - Base.metadata.create_all(db.engine) + + BirthNames.create(db.engine) session = db.session() - with gzip.open(config.basedir + '/data/birth_names.csv.gz') as f: + with gzip.open(config.get("BASE_DIR") + '/data/birth_names.csv.gz') as f: bb_csv = csv.reader(f) for i, (state, year, name, gender, num) in enumerate(bb_csv): if i == 0: @@ -68,24 +77,27 @@ def load_examples(sample): if num == "NA": num = 0 ds = str(year) + '-01-01' - session.add( - BirthNames( - state=state, year=year, - ds=ds, - name=name, num=num, gender=gender)) - if i % 1000 == 0: + db.engine.execute( + BirthNames.insert(), + state=state, + year=year, + ds=ds, + name=name, num=num, gender=gender) + if i % 5000 == 0: print("{} loaded out of 82527 rows".format(i)) session.commit() session.commit() if sample and i>1000: break print("Done loading table!") print("-" * 80) + print("Creating database reference") DB = models.Database dbobj = session.query(DB).filter_by(database_name='main').first() if not dbobj: dbobj = DB(database_name="main") - dbobj.sqlalchemy_uri = config.SQLALCHEMY_DATABASE_URI + print config.get("SQLALCHEMY_DATABASE_URI") + dbobj.sqlalchemy_uri = config.get("SQLALCHEMY_DATABASE_URI") session.add(dbobj) session.commit() @@ -99,10 +111,10 @@ def load_examples(sample): obj.database = dbobj obj.columns = [models.TableColumn( column_name="num", sum=True, type="INTEGER")] - obj.fetch_metadata() models.Table session.add(obj) session.commit() + obj.fetch_metadata() tbl = obj print("Creating some slices") @@ -121,7 +133,7 @@ def load_examples(sample): "groupby": [], "metric": 'sum__num', "metrics": ["sum__num"], - "row_limit": config.ROW_LIMIT, + "row_limit": config.get("ROW_LIMIT"), "since": "100 years", "slice_name": slice_name, "until": "now", @@ -245,7 +257,7 @@ The source dataset came from [here](https://github.com/hadley/babynames) datasource_type='table', table=tbl, params=get_slice_json( - slice_name, viz_type="word_cloud", size_from="10", + slice_name, viz_type="word_cloud", size_from="10", groupby=['name'], size_to="70", rotation="square", limit='100')) session.add(slc) diff --git a/panoramix/config.py b/panoramix/config.py index 2466947488..3e5e0dc76a 100644 --- a/panoramix/config.py +++ b/panoramix/config.py @@ -2,7 +2,7 @@ import os from flask_appbuilder.security.manager import AUTH_DB # from flask_appbuilder.security.manager import ( # AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH) -basedir = os.path.abspath(os.path.dirname(__file__)) +BASE_DIR = os.path.abspath(os.path.dirname(__file__)) from dateutil import tz """ @@ -107,10 +107,10 @@ LANGUAGES = { # Image and file configuration # --------------------------------------------------- # The file upload folder, when using models with files -UPLOAD_FOLDER = basedir + '/app/static/uploads/' +UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/' # The image upload folder, when using models with images -IMG_UPLOAD_FOLDER = basedir + '/app/static/uploads/' +IMG_UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/' # The image upload url, when using models with images IMG_UPLOAD_URL = '/static/uploads/' diff --git a/panoramix/models.py b/panoramix/models.py index eb5f3eccd5..0146ea3e6d 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -22,10 +22,12 @@ import sqlparse import requests import textwrap -from panoramix import db, get_session, config, utils +from panoramix import app, db, get_session, utils from panoramix.viz import viz_types from sqlalchemy.ext.declarative import declared_attr +config = app.config + QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration']) @@ -402,6 +404,7 @@ class Table(Model, Queryable, AuditMixinNullable): df=df, duration=datetime.now() - qry_start_dttm, query=sql) def fetch_metadata(self): + table = self.database.get_table(self.table_name) try: table = self.database.get_table(self.table_name) except Exception as e: @@ -673,8 +676,8 @@ class Datasource(Model, AuditMixin, Queryable): qry_start_dttm = datetime.now() # add tzinfo to native datetime with config - from_dttm = from_dttm.replace(tzinfo=config.DRUID_TZ) - to_dttm = to_dttm.replace(tzinfo=config.DRUID_TZ) + from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ")) + to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ")) query_str = "" aggregations = { diff --git a/panoramix/templates/appbuilder/baselayout.html b/panoramix/templates/appbuilder/baselayout.html index 05e508ef99..bace4f4293 100644 --- a/panoramix/templates/appbuilder/baselayout.html +++ b/panoramix/templates/appbuilder/baselayout.html @@ -2,8 +2,8 @@ {% import 'appbuilder/baselib.html' as baselib %} {% block body %} - {% include 'appbuilder/general/confirm.html' %} - {% include 'appbuilder/general/alert.html' %} + {% include 'appbuilder/general/confirm.html' %} + {% include 'appbuilder/general/alert.html' %} {% block navbar %}
@@ -11,6 +11,8 @@
{% endblock %} + {% block uncontained %}{% endblock %} +
{% block messages %} @@ -24,8 +26,6 @@ {% block content_fluid %} {% endblock %}
- {% block uncontained %} - {% endblock %} {% block footer %}
diff --git a/panoramix/templates/index.html b/panoramix/templates/index.html index 4805775c6a..fc56a8fa9d 100644 --- a/panoramix/templates/index.html +++ b/panoramix/templates/index.html @@ -127,7 +127,7 @@
Generic placeholder image -

Galery

+

Gallery

Navigate through the growing set of visualizations

diff --git a/panoramix/templates/panoramix/models/database/macros.html b/panoramix/templates/panoramix/models/database/macros.html index 877b318888..8e5ba5ee63 100644 --- a/panoramix/templates/panoramix/models/database/macros.html +++ b/panoramix/templates/panoramix/models/database/macros.html @@ -5,7 +5,7 @@ $("#testconn").click(function() { var url = "/panoramix/testconn"; $.ajax({ - method: "GET", + method: "POST", url: url, data: { uri: $("#sqlalchemy_uri").val() } }).done(function() { diff --git a/panoramix/views.py b/panoramix/views.py index 7c32ef04ab..a764564819 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -11,7 +11,9 @@ from pydruid.client import doublesum from sqlalchemy import create_engine from wtforms.validators import ValidationError -from panoramix import appbuilder, db, models, viz, utils, app, config +from panoramix import appbuilder, db, models, viz, utils, app + +config = app.config def validate_json(form, field): @@ -300,7 +302,7 @@ class Panoramix(BaseView): try: resp = self.render_template("panoramix/viz.html", viz=obj) except Exception as e: - if config.DEBUG: + if config.get("DEBUG"): raise(e) return Response( str(e), @@ -326,12 +328,11 @@ class Panoramix(BaseView): return "SUCCESS" @has_access - @expose("/testconn/") + @expose("/testconn", methods=["POST"]) def testconn(self): try: - db = create_engine(request.args.get('uri')) - for i in range(15): - request.args.get('uri') + uri = request.form.get('uri') + db = create_engine(uri) db.connect() return "SUCCESS" except Exception as e: diff --git a/panoramix/viz.py b/panoramix/viz.py index b80e8322cd..babc30c04f 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -10,10 +10,12 @@ from werkzeug.urls import Href import numpy as np import pandas as pd -from panoramix import utils, config +from panoramix import app, utils from panoramix.highchart import Highchart, HighchartBubble from panoramix.forms import form_factory +config = app.config + CHART_ARGS = { 'title': None, } @@ -106,7 +108,7 @@ class BaseViz(object): granularity).total_seconds() * 1000 limit = int(args.get("limit", 0)) row_limit = int( - args.get("row_limit", config.ROW_LIMIT)) + args.get("row_limit", config.get("ROW_LIMIT"))) since = args.get("since", "1 year ago") from_dttm = utils.parse_human_datetime(since) if from_dttm > datetime.now(): diff --git a/requirements.txt b/requirements.txt index 9d3a44349a..293125cd81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ +coverage flask -flask-migrate flask-appbuilder +flask-migrate +flask-testing gunicorn markdown mysql-python