Starting over
|
@ -1 +1,2 @@
|
|||
*.pyc
|
||||
tmp
|
||||
|
|
86
base.html
|
@ -1,86 +0,0 @@
|
|||
{% import 'admin/layout.html' as layout with context -%}
|
||||
{% import 'admin/static.html' as admin_static with context %}
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>{% block title %}{% if admin_view.category %}{{ admin_view.category }} - {% endif %}{{ admin_view.name }} - {{ admin_view.admin.name }}{% endblock %}</title>
|
||||
{% block head_meta %}
|
||||
<meta charset="UTF-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
{% endblock %}
|
||||
{% block head_css %}
|
||||
<link href="{{ admin_static.url(filename='bootstrap/bootstrap3/css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='bootstrap-theme.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='main.css') }}" rel="stylesheet">
|
||||
<link href="{{ admin_static.url(filename='admin/css/bootstrap3/admin.css') }}" rel="stylesheet">
|
||||
<link rel="stylesheet" href="{{url_for('static', filename='select2-bootstrap.css')}}">
|
||||
<link rel="stylesheet" href="{{url_for('static', filename='select2.min.css')}}">
|
||||
{% endblock %}
|
||||
{% block head %}
|
||||
{% endblock %}
|
||||
{% block head_tail %}
|
||||
{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
{% block page_body %}
|
||||
<div class="container">
|
||||
<nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
|
||||
<div class="container">
|
||||
<!-- Brand and toggle get grouped for better mobile display -->
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#admin-navbar-collapse">
|
||||
<span class="sr-only">Toggle navigation</span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
{% block brand %}
|
||||
<a class="navbar-brand" href="#">{{ admin_view.admin.name }}</a>
|
||||
{% endblock %}
|
||||
</div>
|
||||
<!-- navbar content -->
|
||||
<div class="collapse navbar-collapse" id="admin-navbar-collapse">
|
||||
{% block main_menu %}
|
||||
<ul class="nav navbar-nav">
|
||||
{{ layout.menu() }}
|
||||
</ul>
|
||||
{% endblock %}
|
||||
|
||||
{% block menu_links %}
|
||||
<ul class="nav navbar-right">
|
||||
{{ layout.menu_links() }}
|
||||
</ul>
|
||||
{% endblock %}
|
||||
{% block access_control %}
|
||||
{% endblock %}
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
{% block messages %}
|
||||
{{ layout.messages() }}
|
||||
{% endblock %}
|
||||
|
||||
{% set render_ctx = h.resolve_ctx() %}
|
||||
|
||||
{% block body %}{% endblock %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block tail_js %}
|
||||
<script src="{{ admin_static.url(filename='vendor/jquery-2.1.1.min.js') }}" type="text/javascript"></script>
|
||||
<script src="{{ admin_static.url(filename='bootstrap/bootstrap3/js/bootstrap.min.js') }}" type="text/javascript"></script>
|
||||
<script src="{{ admin_static.url(filename='vendor/moment-2.8.4.min.js') }}" type="text/javascript"></script>
|
||||
<script src="{{ admin_static.url(filename='vendor/select2/select2.min.js') }}" type="text/javascript"></script>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block tail %}
|
||||
{% endblock %}
|
||||
</body>
|
||||
</html>
|
1
init.sh
|
@ -1 +0,0 @@
|
|||
export PYTHONPATH=/home/maxime_beauchemin/code/panoramix:/home/maxime_beauchemin/code/pydruid
|
|
@ -0,0 +1,9 @@
|
|||
*.pyc
|
||||
app.db
|
||||
./tmp
|
||||
./build/*
|
||||
./.idea
|
||||
./.idea/*
|
||||
env
|
||||
venv
|
||||
*.sublime*
|
|
@ -0,0 +1,15 @@
|
|||
Base Skeleton to start your application using Flask-AppBuilder
|
||||
--------------------------------------------------------------
|
||||
|
||||
- Install it::
|
||||
|
||||
pip install flask-appbuilder
|
||||
git clone https://github.com/dpgaspar/Flask-AppBuilder-Skeleton.git
|
||||
|
||||
- Run it::
|
||||
|
||||
fabmanager run
|
||||
|
||||
|
||||
That's it!!
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# TODO
|
||||
* Default URL params per datasource
|
||||
* Get config metrics to work
|
251
panoramix/app.py
|
@ -1,251 +0,0 @@
|
|||
from dateutil.parser import parse
|
||||
from datetime import timedelta
|
||||
from flask import Flask, request, Blueprint, url_for, Markup
|
||||
from flask.ext.sqlalchemy import SQLAlchemy
|
||||
from flask.ext.admin import Admin, BaseView, expose, AdminIndexView
|
||||
from panoramix import settings, viz, models
|
||||
from flask_bootstrap import Bootstrap
|
||||
from wtforms import Form, SelectMultipleField, SelectField, TextField
|
||||
from wtforms.fields import Field
|
||||
import pandas as pd
|
||||
from flask_admin.contrib import sqla
|
||||
|
||||
|
||||
pd.set_option('display.max_colwidth', -1)
|
||||
|
||||
client = settings.get_pydruid_client()
|
||||
|
||||
|
||||
class OmgWtForm(Form):
|
||||
field_order = (
|
||||
'viz_type', 'granularity', 'since', 'group_by', 'limit')
|
||||
def fields(self):
|
||||
fields = []
|
||||
for field in self.field_order:
|
||||
if hasattr(self, field):
|
||||
obj = getattr(self, field)
|
||||
if isinstance(obj, Field):
|
||||
fields.append(getattr(self, field))
|
||||
return fields
|
||||
|
||||
|
||||
def form_factory(datasource, form_args=None):
|
||||
grain = ['all', 'none', 'minute', 'hour', 'day']
|
||||
limits = [0, 5, 10, 25, 50, 100, 500]
|
||||
|
||||
if form_args:
|
||||
limit = form_args.get("limit")
|
||||
try:
|
||||
limit = int(limit)
|
||||
if limit not in limits:
|
||||
limits.append(limit)
|
||||
limits = sorted(limits)
|
||||
except:
|
||||
pass
|
||||
|
||||
class QueryForm(OmgWtForm):
|
||||
viz_type = SelectField(
|
||||
'Viz',
|
||||
choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()])
|
||||
metric = SelectField(
|
||||
'Metric', choices=[(m, m) for m in datasource.metrics])
|
||||
groupby = SelectMultipleField(
|
||||
'Group by', choices=[
|
||||
(s, s) for s in datasource.groupby_column_names])
|
||||
granularity = SelectField(
|
||||
'Time Granularity', choices=[(g, g) for g in grain])
|
||||
since = SelectField(
|
||||
'Since', choices=[(s, s) for s in settings.since_l.keys()],
|
||||
default="all")
|
||||
limit = SelectField(
|
||||
'Limit', choices=[(s, s) for s in limits])
|
||||
for i in range(10):
|
||||
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
|
||||
'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names]))
|
||||
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
|
||||
'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]]))
|
||||
setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super"))
|
||||
return QueryForm
|
||||
|
||||
"""
|
||||
bp = Blueprint(
|
||||
'panoramix', __name__,
|
||||
template_folder='templates',
|
||||
static_folder='static')
|
||||
"""
|
||||
|
||||
|
||||
|
||||
app = Flask(__name__)
|
||||
app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI
|
||||
db = SQLAlchemy(app)
|
||||
app.secret_key = "monkeys"
|
||||
#app.register_blueprint(bp, url_prefix='/panoramix')
|
||||
Bootstrap(app)
|
||||
admin = Admin(
|
||||
app, name = "Panoramix",
|
||||
template_mode='bootstrap3')
|
||||
|
||||
|
||||
|
||||
class Datasource(db.Model):
|
||||
__tablename__ = 'datasources'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
datasource_name = db.Column(db.String(256), unique=True)
|
||||
is_featured = db.Column(db.Boolean, default=False)
|
||||
is_hidden = db.Column(db.Boolean, default=False)
|
||||
description = db.Column(db.Text)
|
||||
created_dttm = db.Column(db.DateTime, default=db.func.now())
|
||||
|
||||
@property
|
||||
def metrics(self):
|
||||
return [col.column_name for col in self.columns if not col.groupby]
|
||||
|
||||
@classmethod
|
||||
def latest_metadata(cls, name):
|
||||
results = client.time_boundary(datasource=name)
|
||||
max_time = results[0]['result']['maxTime']
|
||||
max_time = parse(max_time)
|
||||
intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
|
||||
intervals += (max_time + timedelta(seconds=1)).isoformat()
|
||||
segment_metadata = client.segment_metadata(
|
||||
datasource=name,
|
||||
intervals=intervals)
|
||||
return segment_metadata[-1]['columns']
|
||||
|
||||
@classmethod
|
||||
def sync_to_db(cls, name):
|
||||
datasource = cls.query.filter_by(datasource_name=name).first()
|
||||
if not datasource:
|
||||
db.session.add(cls(datasource_name=name))
|
||||
cols = cls.latest_metadata(name)
|
||||
for col in cols:
|
||||
col_obj = Column.query.filter_by(datasource_name=name, column_name=col).first()
|
||||
datatype = cols[col]['type']
|
||||
if not col_obj:
|
||||
col_obj = Column(datasource_name=name, column_name=col)
|
||||
db.session.add(col_obj)
|
||||
if datatype == "STRING":
|
||||
col_obj.groupby = True
|
||||
col_obj.filterable = True
|
||||
if col_obj:
|
||||
col_obj.type = cols[col]['type']
|
||||
|
||||
db.session.commit()
|
||||
|
||||
@property
|
||||
def column_names(self):
|
||||
return sorted([c.column_name for c in self.columns])
|
||||
|
||||
@property
|
||||
def groupby_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.groupby])
|
||||
|
||||
@property
|
||||
def filterable_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.filterable])
|
||||
|
||||
|
||||
|
||||
class Column(db.Model):
|
||||
__tablename__ = 'columns'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
datasource_name = db.Column(
|
||||
db.String(256),
|
||||
db.ForeignKey('datasources.datasource_name'))
|
||||
column_name = db.Column(db.String(256))
|
||||
is_active = db.Column(db.Boolean, default=True)
|
||||
type = db.Column(db.String(32))
|
||||
groupby = db.Column(db.Boolean, default=False)
|
||||
count_distinct = db.Column(db.Boolean, default=False)
|
||||
sum = db.Column(db.Boolean, default=False)
|
||||
max = db.Column(db.Boolean, default=False)
|
||||
min = db.Column(db.Boolean, default=False)
|
||||
filterable = db.Column(db.Boolean, default=False)
|
||||
datasource = db.relationship('Datasource',
|
||||
backref=db.backref('columns', lazy='dynamic'))
|
||||
|
||||
def __repr__(self):
|
||||
return self.column_name
|
||||
|
||||
|
||||
class JsUdf(db.Model):
|
||||
__tablename__ = 'udfs'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
datasource_name = db.Column(
|
||||
db.String(256),
|
||||
db.ForeignKey('datasources.datasource_name'))
|
||||
udf_name = db.Column(db.String(256))
|
||||
column_list = db.Column(db.String(1024))
|
||||
code = db.Column(db.Text)
|
||||
datasource = db.relationship('Datasource',
|
||||
backref=db.backref('udfs', lazy='dynamic'))
|
||||
|
||||
|
||||
def datasource_link(v, c, m, p):
|
||||
url = '/admin/datasourceview/datasource/{}/'.format(m.datasource_name)
|
||||
return Markup('<a href="{url}">{m.datasource_name}</a>'.format(**locals()))
|
||||
|
||||
|
||||
class DatasourceAdmin(sqla.ModelView):
|
||||
inline_models = (Column, JsUdf,)
|
||||
column_formatters = dict(datasource_name=datasource_link)
|
||||
|
||||
|
||||
class DatasourceView(BaseView):
|
||||
@expose('/')
|
||||
def index(self):
|
||||
return ""
|
||||
@expose("/datasource/<datasource_name>/")
|
||||
def datasource(self, datasource_name):
|
||||
viz_type = request.args.get("viz_type", "table")
|
||||
datasource = (
|
||||
Datasource
|
||||
.query
|
||||
.filter_by(datasource_name=datasource_name)
|
||||
.first()
|
||||
)
|
||||
obj = viz.viz_types[viz_type](
|
||||
datasource,
|
||||
form_class=form_factory(datasource, request.args),
|
||||
form_data=request.args,
|
||||
admin_view=self)
|
||||
if obj.df is None or obj.df.empty:
|
||||
return obj.render_no_data()
|
||||
return obj.render()
|
||||
|
||||
|
||||
@expose("/datasources/")
|
||||
def datasources(self):
|
||||
import requests
|
||||
import json
|
||||
endpoint = (
|
||||
"http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
|
||||
"{COORDINATOR_BASE_ENDPOINT}/datasources"
|
||||
).format(**settings.__dict__)
|
||||
datasources = json.loads(requests.get(endpoint).text)
|
||||
for datasource in datasources:
|
||||
Datasource.sync_to_db(datasource)
|
||||
|
||||
return json.dumps(datasources, indent=4)
|
||||
|
||||
|
||||
@expose("/datasource_metadata/<name>/")
|
||||
def datasource_metadata(name):
|
||||
import requests
|
||||
import json
|
||||
endpoint = (
|
||||
"http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
|
||||
"{COORDINATOR_BASE_ENDPOINT}/datasource"
|
||||
).format(**settings.__dict__)
|
||||
|
||||
return str(datasources)
|
||||
|
||||
admin.add_view(DatasourceView(name="Datasource"))
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
db.create_all()
|
||||
admin.add_view(DatasourceAdmin(Datasource, db.session, name="Datasources"))
|
||||
app.debug = True
|
||||
app.run(host='0.0.0.0', port=settings.FLASK_APP_PORT)
|
|
@ -0,0 +1,34 @@
|
|||
import logging
|
||||
from flask import Flask
|
||||
from flask.ext.appbuilder import SQLA, AppBuilder
|
||||
|
||||
"""
|
||||
Logging configuration
|
||||
"""
|
||||
|
||||
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
app = Flask(__name__)
|
||||
app.config.from_object('config')
|
||||
db = SQLA(app)
|
||||
appbuilder = AppBuilder(
|
||||
app, db.session, base_template='panoramix/base.html')
|
||||
#appbuilder.app_name = 'Panoramix'
|
||||
|
||||
|
||||
"""
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy import event
|
||||
|
||||
#Only include this for SQLLite constraints
|
||||
@event.listens_for(Engine, "connect")
|
||||
def set_sqlite_pragma(dbapi_connection, connection_record):
|
||||
# Will force sqllite contraint foreign keys
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
"""
|
||||
|
||||
from app import views
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
from flask.ext.appbuilder import Model
|
||||
from datetime import datetime, timedelta
|
||||
from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn, ImageColumn
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean
|
||||
from sqlalchemy.orm import relationship
|
||||
from app import db, utils
|
||||
from dateutil.parser import parse
|
||||
"""
|
||||
|
||||
You can use the extra Flask-AppBuilder fields and Mixin's
|
||||
|
||||
AuditMixin will add automatic timestamp of created and modified by who
|
||||
|
||||
|
||||
"""
|
||||
client = utils.get_pydruid_client()
|
||||
|
||||
class Datasource(Model, AuditMixin):
|
||||
__tablename__ = 'datasources'
|
||||
id = Column(Integer, primary_key=True)
|
||||
datasource_name = Column(String(256), unique=True)
|
||||
is_featured = Column(Boolean, default=False)
|
||||
is_hidden = Column(Boolean, default=False)
|
||||
description = Column(Text)
|
||||
columns = relationship('Column', backref='datasource')
|
||||
udfs = relationship('JavascriptUdf', backref='datasource')
|
||||
|
||||
@property
|
||||
def metrics(self):
|
||||
return [col.column_name for col in self.columns if not col.groupby]
|
||||
|
||||
def __repr__(self):
|
||||
return self.datasource_name
|
||||
|
||||
@property
|
||||
def datasource_link(self):
|
||||
url = "/panoramix/datasource/{}/".format(self.datasource_name)
|
||||
return '<a href="{url}">{self.datasource_name}</a>'.format(**locals())
|
||||
|
||||
@classmethod
|
||||
def latest_metadata(cls, name):
|
||||
results = client.time_boundary(datasource=name)
|
||||
max_time = results[0]['result']['maxTime']
|
||||
max_time = parse(max_time)
|
||||
intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
|
||||
intervals += (max_time + timedelta(seconds=1)).isoformat()
|
||||
segment_metadata = client.segment_metadata(
|
||||
datasource=name,
|
||||
intervals=intervals)
|
||||
return segment_metadata[-1]['columns']
|
||||
|
||||
@classmethod
|
||||
def sync_to_db(cls, name):
|
||||
datasource = db.session.query(cls).filter_by(datasource_name=name).first()
|
||||
if not datasource:
|
||||
db.session.add(cls(datasource_name=name))
|
||||
cols = cls.latest_metadata(name)
|
||||
for col in cols:
|
||||
col_obj = (
|
||||
db.session
|
||||
.query(Column)
|
||||
.filter_by(datasource_name=name, column_name=col)
|
||||
.first()
|
||||
)
|
||||
datatype = cols[col]['type']
|
||||
if not col_obj:
|
||||
col_obj = Column(datasource_name=name, column_name=col)
|
||||
db.session.add(col_obj)
|
||||
if datatype == "STRING":
|
||||
col_obj.groupby = True
|
||||
col_obj.filterable = True
|
||||
if col_obj:
|
||||
col_obj.type = cols[col]['type']
|
||||
|
||||
db.session.commit()
|
||||
|
||||
@property
|
||||
def column_names(self):
|
||||
return sorted([c.column_name for c in self.columns])
|
||||
|
||||
@property
|
||||
def groupby_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.groupby])
|
||||
|
||||
@property
|
||||
def filterable_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.filterable])
|
||||
|
||||
|
||||
class JavascriptUdf(Model, AuditMixin):
|
||||
__tablename__ = 'udfs'
|
||||
id = Column(Integer, primary_key=True)
|
||||
datasource_name = Column(
|
||||
String(256),
|
||||
ForeignKey('datasources.datasource_name'))
|
||||
udf_name = Column(String(256))
|
||||
column_list = Column(String(1024))
|
||||
code = Column(Text)
|
||||
|
||||
def __repr__(self):
|
||||
return self.udf_name
|
||||
|
||||
|
||||
class Column(Model, AuditMixin):
|
||||
__tablename__ = 'columns'
|
||||
id = Column(Integer, primary_key=True)
|
||||
datasource_name = Column(
|
||||
String(256),
|
||||
ForeignKey('datasources.datasource_name'))
|
||||
column_name = Column(String(256))
|
||||
is_active = Column(Boolean, default=True)
|
||||
type = Column(String(32))
|
||||
groupby = Column(Boolean, default=False)
|
||||
count_distinct = Column(Boolean, default=False)
|
||||
sum = Column(Boolean, default=False)
|
||||
max = Column(Boolean, default=False)
|
||||
min = Column(Boolean, default=False)
|
||||
filterable = Column(Boolean, default=False)
|
||||
|
||||
def __repr__(self):
|
||||
return self.column_name
|
||||
|
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 24 KiB |
Before Width: | Height: | Size: 165 KiB After Width: | Height: | Size: 165 KiB |
Before Width: | Height: | Size: 1.5 MiB After Width: | Height: | Size: 1.5 MiB |
Before Width: | Height: | Size: 121 KiB After Width: | Height: | Size: 121 KiB |
Before Width: | Height: | Size: 73 KiB After Width: | Height: | Size: 73 KiB |
|
@ -0,0 +1,2 @@
|
|||
{% extends "appbuilder/baselayout.html" %}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{% extends "appbuilder/baselayout.html" %}
|
||||
|
||||
{% block head_css %}
|
||||
{{super()}}
|
||||
<link rel="shortcut icon" href="{{ url_for('static', filename='chaudron.png') }}">
|
||||
<style>
|
||||
</style>
|
||||
{% endblock %}
|
|
@ -1,5 +1,5 @@
|
|||
{% extends "panoramix/base.html" %}
|
||||
{% block styles %}
|
||||
{% block head_css %}
|
||||
{{super()}}
|
||||
<style>
|
||||
form .row {
|
||||
|
@ -12,22 +12,22 @@ form .col {
|
|||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
{% block body %}
|
||||
{% block content %}
|
||||
<div class="container">
|
||||
<div class="col-md-3">
|
||||
<h3>
|
||||
{{ datasource.datasource_name }}
|
||||
<a href="/admin/datasource/edit/?id={{ datasource.id }}"><span class="glyphicon glyphicon-edit"></span></a>
|
||||
<a href="/datasourcemodelview/edit/{{ datasource.id }}"><span class="glyphicon glyphicon-edit"></span></a>
|
||||
</h3>
|
||||
|
||||
<hr>
|
||||
<form method="GET">
|
||||
<div>{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control") }}</div>
|
||||
<div>{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control select2") }}</div>
|
||||
<div>{{ form.metric.label }}: {{ form.metric(class_="form-control select2") }}</div>
|
||||
<div>{{ form.granularity.label }}: {{ form.granularity(class_="form-control") }}</div>
|
||||
<div>{{ form.since.label }}: {{ form.since(class_="form-control") }}</div>
|
||||
<div>{{ form.groupby.label }}: {{ form.groupby(class_="form-control") }}</div>
|
||||
<div>{{ form.limit.label }}: {{ form.limit(class_="form-control") }}</div>
|
||||
<div>{{ form.granularity.label }}: {{ form.granularity(class_="form-control select2") }}</div>
|
||||
<div>{{ form.since.label }}: {{ form.since(class_="form-control select2") }}</div>
|
||||
<div>{{ form.groupby.label }}: {{ form.groupby(class_="form-control select2") }}</div>
|
||||
<div>{{ form.limit.label }}: {{ form.limit(class_="form-control select2") }}</div>
|
||||
<hr>
|
||||
<h4>Filters</h4>
|
||||
<div id="filters">
|
||||
|
@ -78,12 +78,12 @@ form .col {
|
|||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block tail %}
|
||||
{% block tail_js %}
|
||||
{{ super() }}
|
||||
<script>
|
||||
$( document ).ready(function() {
|
||||
//`:$(".select2").select2();
|
||||
// $(".select2_tags").select2({tags: true});
|
||||
$(".select2").select2();
|
||||
$(".select2_tags").select2({tags: true});
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
|
@ -0,0 +1,18 @@
|
|||
import config
|
||||
from datetime import timedelta
|
||||
|
||||
since_l = {
|
||||
'1hour': timedelta(hours=1),
|
||||
'1day': timedelta(days=1),
|
||||
'7days': timedelta(days=7),
|
||||
'28days': timedelta(days=28),
|
||||
'all': timedelta(days=365*100)
|
||||
}
|
||||
|
||||
def get_pydruid_client():
|
||||
from pydruid import client
|
||||
return client.PyDruid(
|
||||
"http://{0}:{1}/".format(config.DRUID_HOST, config.DRUID_PORT),
|
||||
config.DRUID_BASE_ENDPOINT)
|
||||
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
from flask import request, redirect, flash
|
||||
from flask.ext.appbuilder.models.sqla.interface import SQLAInterface
|
||||
from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose
|
||||
from app import appbuilder, db, models, viz, utils
|
||||
import config
|
||||
from wtforms import Form, SelectMultipleField, SelectField, TextField
|
||||
from wtforms.fields import Field
|
||||
from datetime import timedelta
|
||||
|
||||
class OmgWtForm(Form):
|
||||
field_order = (
|
||||
'viz_type', 'granularity', 'since', 'group_by', 'limit')
|
||||
def fields(self):
|
||||
fields = []
|
||||
for field in self.field_order:
|
||||
if hasattr(self, field):
|
||||
obj = getattr(self, field)
|
||||
if isinstance(obj, Field):
|
||||
fields.append(getattr(self, field))
|
||||
return fields
|
||||
|
||||
|
||||
def form_factory(datasource, form_args=None):
|
||||
grain = ['all', 'none', 'minute', 'hour', 'day']
|
||||
limits = [0, 5, 10, 25, 50, 100, 500]
|
||||
|
||||
if form_args:
|
||||
limit = form_args.get("limit")
|
||||
try:
|
||||
limit = int(limit)
|
||||
if limit not in limits:
|
||||
limits.append(limit)
|
||||
limits = sorted(limits)
|
||||
except:
|
||||
pass
|
||||
|
||||
class QueryForm(OmgWtForm):
|
||||
viz_type = SelectField(
|
||||
'Viz',
|
||||
choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()])
|
||||
metric = SelectField(
|
||||
'Metric', choices=[(m, m) for m in datasource.metrics])
|
||||
groupby = SelectMultipleField(
|
||||
'Group by', choices=[
|
||||
(s, s) for s in datasource.groupby_column_names])
|
||||
granularity = SelectField(
|
||||
'Time Granularity', choices=[(g, g) for g in grain])
|
||||
since = SelectField(
|
||||
'Since', choices=[(s, s) for s in utils.since_l.keys()],
|
||||
default="all")
|
||||
limit = SelectField(
|
||||
'Limit', choices=[(s, s) for s in limits])
|
||||
for i in range(10):
|
||||
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
|
||||
'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names]))
|
||||
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
|
||||
'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]]))
|
||||
setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super"))
|
||||
return QueryForm
|
||||
|
||||
|
||||
class ColumnInlineView(CompactCRUDMixin, ModelView):
|
||||
datamodel = SQLAInterface(models.Column)
|
||||
edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
|
||||
list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
|
||||
can_delete = False
|
||||
appbuilder.add_view_no_menu(ColumnInlineView)
|
||||
|
||||
class JavascriptUdfInlineView(CompactCRUDMixin, ModelView):
|
||||
datamodel = SQLAInterface(models.JavascriptUdf)
|
||||
edit_columns = ['udf_name', 'column_list', 'code']
|
||||
appbuilder.add_view_no_menu(JavascriptUdfInlineView)
|
||||
|
||||
|
||||
class DatasourceModelView(ModelView):
|
||||
datamodel = SQLAInterface(models.Datasource)
|
||||
list_columns = ['datasource_link', 'is_featured' ]
|
||||
related_views = [ColumnInlineView, JavascriptUdfInlineView]
|
||||
edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden']
|
||||
page_size = 100
|
||||
|
||||
|
||||
appbuilder.add_view(
|
||||
DatasourceModelView,
|
||||
"Datasources",
|
||||
icon="fa-cube",
|
||||
category_icon='fa-envelope')
|
||||
|
||||
|
||||
class Panoramix(BaseView):
|
||||
@expose("/datasource/<datasource_name>/")
|
||||
def datasource(self, datasource_name):
|
||||
viz_type = request.args.get("viz_type", "table")
|
||||
datasource = (
|
||||
db.session
|
||||
.query(models.Datasource)
|
||||
.filter_by(datasource_name=datasource_name)
|
||||
.first()
|
||||
)
|
||||
obj = viz.viz_types[viz_type](
|
||||
datasource,
|
||||
form_class=form_factory(datasource, request.args),
|
||||
form_data=request.args, view=self)
|
||||
if obj.df is None or obj.df.empty:
|
||||
return obj.render_no_data()
|
||||
return obj.render()
|
||||
|
||||
|
||||
@expose("/refresh_datasources/")
|
||||
def datasources(self):
|
||||
import requests
|
||||
import json
|
||||
endpoint = (
|
||||
"http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
|
||||
"{COORDINATOR_BASE_ENDPOINT}/datasources"
|
||||
).format(**config.__dict__)
|
||||
datasources = json.loads(requests.get(endpoint).text)
|
||||
for datasource in datasources:
|
||||
models.Datasource.sync_to_db(datasource)
|
||||
flash("Refreshed metadata from Druid!", 'info')
|
||||
return redirect("/datasourcemodelview/list/")
|
||||
|
||||
appbuilder.add_view_no_menu(Panoramix)
|
||||
appbuilder.add_link(
|
||||
"Refresh Metadata",
|
||||
href='/panoramix/refresh_datasources/',
|
||||
category='Admin',
|
||||
icon="fa-cogs")
|
||||
db.create_all()
|
|
@ -3,9 +3,10 @@ from datetime import datetime
|
|||
from flask import render_template, flash
|
||||
import pandas as pd
|
||||
from pandas_highcharts.core import serialize
|
||||
from panoramix import settings
|
||||
from pydruid.utils import aggregators as agg
|
||||
from collections import OrderedDict
|
||||
from app import utils
|
||||
import config
|
||||
|
||||
|
||||
CHART_ARGS = {
|
||||
|
@ -18,13 +19,13 @@ CHART_ARGS = {
|
|||
class BaseViz(object):
|
||||
verbose_name = "Base Viz"
|
||||
template = "panoramix/datasource.html"
|
||||
def __init__(self, datasource, form_class, form_data, admin_view):
|
||||
def __init__(self, datasource, form_class, form_data, view):
|
||||
self.datasource = datasource
|
||||
self.form_class = form_class
|
||||
self.form_data = form_data
|
||||
self.metric = form_data.get('metric')
|
||||
self.admin_view = admin_view
|
||||
self.df = self.bake_query()
|
||||
self.view = view
|
||||
if self.df is not None:
|
||||
self.df.timestamp = pd.to_datetime(self.df.timestamp)
|
||||
self.df_prep()
|
||||
|
@ -68,9 +69,9 @@ class BaseViz(object):
|
|||
granularity = args.get("granularity")
|
||||
metric = "count"
|
||||
limit = int(
|
||||
args.get("limit", settings.ROW_LIMIT)) or settings.ROW_LIMIT
|
||||
args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT
|
||||
since = args.get("since", "all")
|
||||
from_dttm = (datetime.now() - settings.since_l[since]).isoformat()
|
||||
from_dttm = (datetime.now() - utils.since_l[since]).isoformat()
|
||||
d = {
|
||||
'datasource': ds.datasource_name,
|
||||
'granularity': granularity or 'all',
|
||||
|
@ -92,7 +93,7 @@ class BaseViz(object):
|
|||
return d
|
||||
|
||||
def bake_query(self):
|
||||
client = settings.get_pydruid_client()
|
||||
client = utils.get_pydruid_client()
|
||||
client.groupby(**self.query_obj())
|
||||
return client.export_pandas()
|
||||
|
||||
|
@ -108,7 +109,7 @@ class BaseViz(object):
|
|||
|
||||
def render(self, *args, **kwargs):
|
||||
form = self.form_class(self.form_data)
|
||||
return self.admin_view.render(
|
||||
return self.view.render_template(
|
||||
self.template, form=form, viz=self, datasource=self.datasource,
|
||||
*args, **kwargs)
|
||||
|
||||
|
@ -159,29 +160,30 @@ class TimeSeriesViz(HighchartsViz):
|
|||
"""
|
||||
Doing a 2 phase query where we limit the number of series.
|
||||
"""
|
||||
client = settings.get_pydruid_client()
|
||||
client = utils.get_pydruid_client()
|
||||
qry = self.query_obj()
|
||||
qry['granularity'] = "all"
|
||||
client.groupby(**qry)
|
||||
df = client.export_pandas()
|
||||
dims = qry['dimensions']
|
||||
filters = []
|
||||
for index, row in df.iterrows():
|
||||
fields = []
|
||||
for dim in dims:
|
||||
f = Filter.build_filter(Dimension(dim) == row[dim])
|
||||
fields.append(f)
|
||||
if len(fields) > 1:
|
||||
filters.append(Filter.build_filter(Filter(type="and", fields=fields)))
|
||||
elif fields:
|
||||
filters.append(fields[0])
|
||||
if not df is None:
|
||||
dims = qry['dimensions']
|
||||
filters = []
|
||||
for index, row in df.iterrows():
|
||||
fields = []
|
||||
for dim in dims:
|
||||
f = Filter.build_filter(Dimension(dim) == row[dim])
|
||||
fields.append(f)
|
||||
if len(fields) > 1:
|
||||
filters.append(Filter.build_filter(Filter(type="and", fields=fields)))
|
||||
elif fields:
|
||||
filters.append(fields[0])
|
||||
|
||||
qry = self.query_obj()
|
||||
if filters:
|
||||
ff = Filter(type="or", fields=filters)
|
||||
qry['filter'] = ff
|
||||
del qry['limit_spec']
|
||||
client.groupby(**qry)
|
||||
qry = self.query_obj()
|
||||
if filters:
|
||||
ff = Filter(type="or", fields=filters)
|
||||
qry['filter'] = ff
|
||||
del qry['limit_spec']
|
||||
client.groupby(**qry)
|
||||
return client.export_pandas()
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[python: **.py]
|
||||
[jinja2: **/templates/**.html]
|
||||
encoding = utf-8
|
|
@ -0,0 +1 @@
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
import os
|
||||
from flask_appbuilder.security.manager import AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH
|
||||
basedir = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
#---------------------------------------------------------
|
||||
# Panoramix specifix config
|
||||
#---------------------------------------------------------
|
||||
ROW_LIMIT = 5000
|
||||
|
||||
DRUID_HOST = '10.181.47.80'
|
||||
DRUID_PORT = 8080
|
||||
DRUID_BASE_ENDPOINT = 'druid/v2'
|
||||
|
||||
COORDINATOR_HOST = '10.168.176.249'
|
||||
COORDINATOR_PORT = '8080'
|
||||
COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1'
|
||||
#---------------------------------------------------------
|
||||
|
||||
# Your App secret key
|
||||
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'
|
||||
|
||||
# The SQLAlchemy connection string.
|
||||
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'app.db')
|
||||
#SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp'
|
||||
#SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp'
|
||||
|
||||
# Flask-WTF flag for CSRF
|
||||
CSRF_ENABLED = True
|
||||
|
||||
#------------------------------
|
||||
# GLOBALS FOR APP Builder
|
||||
#------------------------------
|
||||
# Uncomment to setup Your App name
|
||||
APP_NAME = "Panoramix"
|
||||
|
||||
# Uncomment to setup Setup an App icon
|
||||
#APP_ICON = "static/img/logo.jpg"
|
||||
|
||||
#----------------------------------------------------
|
||||
# AUTHENTICATION CONFIG
|
||||
#----------------------------------------------------
|
||||
# The authentication type
|
||||
# AUTH_OID : Is for OpenID
|
||||
# AUTH_DB : Is for database (username/password()
|
||||
# AUTH_LDAP : Is for LDAP
|
||||
# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server
|
||||
AUTH_TYPE = AUTH_DB
|
||||
|
||||
# Uncomment to setup Full admin role name
|
||||
#AUTH_ROLE_ADMIN = 'Admin'
|
||||
|
||||
# Uncomment to setup Public role name, no authentication needed
|
||||
#AUTH_ROLE_PUBLIC = 'Public'
|
||||
|
||||
# Will allow user self registration
|
||||
#AUTH_USER_REGISTRATION = True
|
||||
|
||||
# The default user self registration role
|
||||
#AUTH_USER_REGISTRATION_ROLE = "Public"
|
||||
|
||||
# When using LDAP Auth, setup the ldap server
|
||||
#AUTH_LDAP_SERVER = "ldap://ldapserver.new"
|
||||
|
||||
# Uncomment to setup OpenID providers example for OpenID authentication
|
||||
#OPENID_PROVIDERS = [
|
||||
# { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' },
|
||||
# { 'name': 'AOL', 'url': 'http://openid.aol.com/<username>' },
|
||||
# { 'name': 'Flickr', 'url': 'http://www.flickr.com/<username>' },
|
||||
# { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }]
|
||||
#---------------------------------------------------
|
||||
# Babel config for translations
|
||||
#---------------------------------------------------
|
||||
# Setup default language
|
||||
BABEL_DEFAULT_LOCALE = 'en'
|
||||
# Your application default translation path
|
||||
BABEL_DEFAULT_FOLDER = 'translations'
|
||||
# The allowed translation for you app
|
||||
LANGUAGES = {
|
||||
'en': {'flag':'gb', 'name':'English'},
|
||||
'pt': {'flag':'pt', 'name':'Portuguese'},
|
||||
'pt_BR': {'flag':'br', 'name': 'Pt Brazil'},
|
||||
'es': {'flag':'es', 'name':'Spanish'},
|
||||
'de': {'flag':'de', 'name':'German'},
|
||||
'zh': {'flag':'cn', 'name':'Chinese'},
|
||||
'ru': {'flag':'ru', 'name':'Russian'}
|
||||
}
|
||||
#---------------------------------------------------
|
||||
# Image and file configuration
|
||||
#---------------------------------------------------
|
||||
# The file upload folder, when using models with files
|
||||
UPLOAD_FOLDER = basedir + '/app/static/uploads/'
|
||||
|
||||
# The image upload folder, when using models with images
|
||||
IMG_UPLOAD_FOLDER = basedir + '/app/static/uploads/'
|
||||
|
||||
# The image upload url, when using models with images
|
||||
IMG_UPLOAD_URL = '/static/uploads/'
|
||||
# Setup image size default is (300, 200, True)
|
||||
#IMG_SIZE = (300, 200, True)
|
||||
|
||||
# Theme configuration
|
||||
# these are located on static/appbuilder/css/themes
|
||||
# you can create your own and easily use them placing them on the same dir structure to override
|
||||
APP_THEME = "bootstrap-theme.css" # default bootstrap
|
||||
#APP_THEME = "cerulean.css"
|
||||
#APP_THEME = "amelia.css"
|
||||
#APP_THEME = "cosmo.css"
|
||||
#APP_THEME = "cyborg.css"
|
||||
#APP_THEME = "flatly.css"
|
||||
#APP_THEME = "journal.css"
|
||||
#APP_THEME = "readable.css"
|
||||
#APP_THEME = "simplex.css"
|
||||
#APP_THEME = "slate.css"
|
||||
#APP_THEME = "spacelab.css"
|
||||
#APP_THEME = "united.css"
|
||||
#APP_THEME = "yeti.css"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
from app import app
|
||||
|
||||
app.run(host='0.0.0.0', port=8081, debug=True)
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
from datetime import timedelta
|
||||
|
||||
FLASK_APP_PORT = 8088
|
||||
|
||||
ROW_LIMIT = 10000
|
||||
SQLALCHEMY_DATABASE_URI = "sqlite:////tmp/panoramix.db"
|
||||
|
||||
DRUID_HOST = '10.181.47.80'
|
||||
DRUID_PORT = 8080
|
||||
DRUID_BASE_ENDPOINT = 'druid/v2'
|
||||
|
||||
COORDINATOR_HOST = '10.168.176.249'
|
||||
COORDINATOR_PORT = '8080'
|
||||
COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1'
|
||||
|
||||
since_l = {
|
||||
'1hour': timedelta(hours=1),
|
||||
'1day': timedelta(days=1),
|
||||
'7days': timedelta(days=7),
|
||||
'28days': timedelta(days=28),
|
||||
'all': timedelta(days=365*100)
|
||||
}
|
||||
|
||||
def get_pydruid_client():
|
||||
from pydruid import client
|
||||
return client.PyDruid(
|
||||
"http://{0}:{1}/".format(DRUID_HOST, DRUID_PORT),
|
||||
DRUID_BASE_ENDPOINT)
|
|
@ -1,2 +0,0 @@
|
|||
{% extends "admin/base.html" %}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
{% extends "index.html" %}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
flask
|
||||
flask-admin
|
||||
flask-bootstrap
|
||||
flask-sqlalchemy
|
||||
pandas
|
||||
pandas-highcharts
|
||||
pydruid
|
||||
python-dateutil
|
||||
requests
|
||||
wtforms
|