Making progress

panoramix/models.py

@@ -269,6 +269,10 @@ class Database(Model, AuditMixinNullable):
         conn.password = self.password
         return str(conn)
 
+    @property
+    def sql_link(self):
+        return '<a href="/panoramix/sql/{}/">SQL</a>'.format(self.id)
+
 
 class SqlaTable(Model, Queryable, AuditMixinNullable):
     type = "table"
@@ -309,6 +313,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
         return (
             "[{self.database}].[{self.table_name}]"
             "(id:{self.id})").format(self=self)
+
     @property
     def full_name(self):
         return "[{self.database}].[{self.table_name}]".format(self=self)
@@ -320,6 +325,18 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
             l.append(self.main_dttm_col)
         return l
 
+    @property
+    def html(self):
+        import pandas as pd
+        t = ((c.column_name, c.type) for c in self.columns)
+        df = pd.DataFrame(t)
+        df.columns = ['field', 'type']
+        return df.to_html(
+            index=False,
+            classes=(
+                "dataframe table table-striped table-bordered "
+                "table-condensed"))
+
     @property
     def name(self):
         return self.table_name

panoramix/static/lib/ace/mode-sql.js

@@ -0,0 +1,92 @@
+define("ace/mode/sql_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var SqlHighlightRules = function() {
+
+    var keywords = (
+        "select|insert|update|delete|from|where|and|or|group|by|order|limit|offset|having|as|case|" +
+        "when|else|end|type|left|right|join|on|outer|desc|asc|union"
+    );
+
+    var builtinConstants = (
+        "true|false|null"
+    );
+
+    var builtinFunctions = (
+        "count|min|max|avg|sum|rank|now|coalesce"
+    );
+
+    var keywordMapper = this.createKeywordMapper({
+        "support.function": builtinFunctions,
+        "keyword": keywords,
+        "constant.language": builtinConstants
+    }, "identifier", true);
+
+    this.$rules = {
+        "start" : [ {
+            token : "comment",
+            regex : "--.*$"
+        },  {
+            token : "comment",
+            start : "/\\*",
+            end : "\\*/"
+        }, {
+            token : "string",           // " string
+            regex : '".*?"'
+        }, {
+            token : "string",           // ' string
+            regex : "'.*?'"
+        }, {
+            token : "constant.numeric", // float
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+        }, {
+            token : keywordMapper,
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+        }, {
+            token : "keyword.operator",
+            regex : "\\+|\\-|\\/|\\/\\/|%|<@>|@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|="
+        }, {
+            token : "paren.lparen",
+            regex : "[\\(]"
+        }, {
+            token : "paren.rparen",
+            regex : "[\\)]"
+        }, {
+            token : "text",
+            regex : "\\s+"
+        } ]
+    };
+    this.normalizeRules();
+};
+
+oop.inherits(SqlHighlightRules, TextHighlightRules);
+
+exports.SqlHighlightRules = SqlHighlightRules;
+});
+
+define("ace/mode/sql",["require","exports","module","ace/lib/oop","ace/mode/text","ace/mode/sql_highlight_rules","ace/range"], function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var SqlHighlightRules = require("./sql_highlight_rules").SqlHighlightRules;
+var Range = require("../range").Range;
+
+var Mode = function() {
+    this.HighlightRules = SqlHighlightRules;
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+
+    this.lineCommentStart = "--";
+
+    this.$id = "ace/mode/sql";
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+
+});

panoramix/static/lib/ace/theme-crimson_editor.js

@@ -0,0 +1,118 @@
+define("ace/theme/crimson_editor",["require","exports","module","ace/lib/dom"], function(require, exports, module) {
+exports.isDark = false;
+exports.cssText = ".ace-crimson-editor .ace_gutter {\
+background: #ebebeb;\
+color: #333;\
+overflow : hidden;\
+}\
+.ace-crimson-editor .ace_gutter-layer {\
+width: 100%;\
+text-align: right;\
+}\
+.ace-crimson-editor .ace_print-margin {\
+width: 1px;\
+background: #e8e8e8;\
+}\
+.ace-crimson-editor {\
+background-color: #FFFFFF;\
+color: rgb(64, 64, 64);\
+}\
+.ace-crimson-editor .ace_cursor {\
+color: black;\
+}\
+.ace-crimson-editor .ace_invisible {\
+color: rgb(191, 191, 191);\
+}\
+.ace-crimson-editor .ace_identifier {\
+color: black;\
+}\
+.ace-crimson-editor .ace_keyword {\
+color: blue;\
+}\
+.ace-crimson-editor .ace_constant.ace_buildin {\
+color: rgb(88, 72, 246);\
+}\
+.ace-crimson-editor .ace_constant.ace_language {\
+color: rgb(255, 156, 0);\
+}\
+.ace-crimson-editor .ace_constant.ace_library {\
+color: rgb(6, 150, 14);\
+}\
+.ace-crimson-editor .ace_invalid {\
+text-decoration: line-through;\
+color: rgb(224, 0, 0);\
+}\
+.ace-crimson-editor .ace_fold {\
+}\
+.ace-crimson-editor .ace_support.ace_function {\
+color: rgb(192, 0, 0);\
+}\
+.ace-crimson-editor .ace_support.ace_constant {\
+color: rgb(6, 150, 14);\
+}\
+.ace-crimson-editor .ace_support.ace_type,\
+.ace-crimson-editor .ace_support.ace_class {\
+color: rgb(109, 121, 222);\
+}\
+.ace-crimson-editor .ace_keyword.ace_operator {\
+color: rgb(49, 132, 149);\
+}\
+.ace-crimson-editor .ace_string {\
+color: rgb(128, 0, 128);\
+}\
+.ace-crimson-editor .ace_comment {\
+color: rgb(76, 136, 107);\
+}\
+.ace-crimson-editor .ace_comment.ace_doc {\
+color: rgb(0, 102, 255);\
+}\
+.ace-crimson-editor .ace_comment.ace_doc.ace_tag {\
+color: rgb(128, 159, 191);\
+}\
+.ace-crimson-editor .ace_constant.ace_numeric {\
+color: rgb(0, 0, 64);\
+}\
+.ace-crimson-editor .ace_variable {\
+color: rgb(0, 64, 128);\
+}\
+.ace-crimson-editor .ace_xml-pe {\
+color: rgb(104, 104, 91);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_selection {\
+background: rgb(181, 213, 255);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_step {\
+background: rgb(252, 255, 0);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_stack {\
+background: rgb(164, 229, 101);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_bracket {\
+margin: -1px 0 0 -1px;\
+border: 1px solid rgb(192, 192, 192);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_active-line {\
+background: rgb(232, 242, 254);\
+}\
+.ace-crimson-editor .ace_gutter-active-line {\
+background-color : #dcdcdc;\
+}\
+.ace-crimson-editor .ace_meta.ace_tag {\
+color:rgb(28, 2, 255);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_selected-word {\
+background: rgb(250, 250, 255);\
+border: 1px solid rgb(200, 200, 250);\
+}\
+.ace-crimson-editor .ace_string.ace_regex {\
+color: rgb(192, 0, 192);\
+}\
+.ace-crimson-editor .ace_indent-guide {\
+background: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAACCAYAAACZgbYnAAAAE0lEQVQImWP4////f4bLly//BwAmVgd1/w11/gAAAABJRU5ErkJggg==\") right repeat-y;\
+}";
+
+exports.cssClass = "ace-crimson-editor";
+
+var dom = require("../lib/dom");
+dom.importCssString(exports.cssText, exports.cssClass);
+});

panoramix/static/panoramix.js

@@ -443,6 +443,62 @@ var px = (function() {
     });
   }
 
+  function initSqlEditorView() {
+    var editor = ace.edit("sql");
+    var textarea = $('#sql').hide();
+    editor.setTheme("ace/theme/crimson_editor");
+    editor.setOptions({
+        minLines: 16,
+        maxLines: Infinity,
+    });
+    editor.getSession().setMode("ace/mode/sql");
+    editor.focus();
+
+    $("select").select2({dropdownAutoWidth : true});
+    function showTableMetadata() {
+      $(".metadata").load('/panoramix/table/' + $("#dbtable").val()  + '/');
+    }
+    $("#dbtable").on("change", showTableMetadata);
+    showTableMetadata();
+    $("#create_view").click(function(){alert("Not implemented");});
+    $(".sqlcontent").show();
+    $("#select_star").click(function(){
+      $.ajax('/panoramix/select_star/' + $("#dbtable").val()  + '/')
+        .done(function(msg){
+          editor.setValue(msg);
+        });
+    });
+    $("#run").click(function() {
+      $('#results').hide(0);
+      $('#loading').show(0);
+      $.ajax({
+        type: "POST",
+        url: '/panoramix/runsql/',
+        data: {
+          'data': JSON.stringify({
+          'database_id': $('#database_id').val(),
+          'sql': editor.getSession().getValue(),
+        })},
+        success: function(data) {
+          $('#loading').hide(0);
+          $('#results').show(0);
+          $('#results').html(data);
+
+          var datatable = $('table.sql_results').DataTable({
+            paging: false,
+            searching: true,
+            aaSorting: [],
+          });
+        },
+        error: function(err, err2) {
+          $('#loading').hide(0);
+          $('#results').show(0);
+          $('#results').html(err.responseText);
+        },
+      });
+    });
+  }
+
   function initDashboardView() {
     var gridster = $(".gridster ul").gridster({
       widget_margins: [5, 5],
@@ -546,5 +602,6 @@ var px = (function() {
     timeFormatFactory: timeFormatFactory,
     color: color(),
     renderSlice: renderSlice,
+    initSqlEditorView: initSqlEditorView,
   }
 })();

panoramix/templates/panoramix/ajah.html

@@ -0,0 +1 @@
+{{ content |safe }}

panoramix/templates/panoramix/dashboard.html

@@ -137,9 +137,6 @@
       $(document).ready(function() {
         px.initDashboardView();
         var dashboard = px.Dashboard({{ dashboard.id }});
-        $('#filters').click( function(){
-          alert(dashboard.readFilters());
-        });
       });
     </script>
 {% endblock %}

panoramix/templates/panoramix/sql.html

@@ -8,8 +8,12 @@
     .topsql {
       height: 250px;
     }
+    .dataTables_filter {
+      padding-top: 5px;
+      padding-right: 5px;
+    }
     .bordered {
-      padding: 5px 10px;
+      padding: 0px 0px;
       border: 1px solid grey;
       border-radius: 5px;
       background-color: #EEE;
@@ -26,8 +30,8 @@
     .fillheight {
       height: 100%;
     }
-    #interactive {
-      padding-top: 10px;
+    .interactions {
+      padding-bottom: 10px;
     }
     #results {
       overflow: auto;
@@ -41,7 +45,23 @@
 {% endblock %}
 
 {% block content %}
+<div class="sqlcontent" style="display: none;">
   <h2>db: [{{ db }}]</h2>
+  <div class="row interactions">
+    <div class="col-xs-7 fillheight">
+      <input type="hidden" id="database_id" value="{{ database_id }}">
+      <button class="btn btn-primary" id="run">Run!</button>
+      <button class="btn btn-default" id="create_view">Create View</button>
+    </div>
+    <div class="col-xs-5 fillheight">
+      <select id="dbtable">
+        {% for table in db.tables %}
+          <option value="{{ table.id }}">{{ table }}</option>
+        {% endfor %}
+      </select>
+      <button class="btn btn-default" id="select_star">SELECT *</button>
+    </div>
+  </div>
   <div class="topsql row">
     <div class="col-xs-7 fillheight">
       <textarea id="sql" class="fillup">SELECT * FROM information_schema.tables;
@@ -54,52 +74,26 @@
     </div>
   </div>
   <div id="interactive">
-    <input type="hidden" id="database_id" value="{{ database_id }}">
-    <button class="btn btn-primary" id="run">Run!</button>
-    <button class="btn btn-default" id="view">Create View</button>
   </div>
   <div id="results_section">
     <hr/>
     <img id="loading" width="25" style="display: none;" src="/static/img/loading.gif">
   </div>
   <div>
-    <div id="results" class="bordered"></div>
+    <div id="results" class="bordered" style="display: none;"></div>
   </div>
+</div>
 {% endblock %}
 
 {% block tail_js %}
 {{ super() }}
 <script src="/static/lib/bootstrap-toggle.min.js"></script>
+<script src="/static/lib/ace/ace.js"></script>
 <script src="/static/lib/dataTables/jquery.dataTables.min.js"></script>
 <script src="/static/lib/dataTables/dataTables.bootstrap.js"></script>
 <script>
   $(document).ready(function() {
-    $("#run").click(function() {
-      $('#results').hide(0);
-      $('#loading').show(0);
-      $.ajax({
-        type: "POST",
-        url: '/panoramix/runsql/',
-        data: {
-          'data': JSON.stringify({
-          'database_id': $('#database_id').val(),
-          'sql': $('#sql').val(),
-        })},
-        success: function(data) {
-          $('#loading').hide(0);
-          $('#results').show(0);
-          $('#results').html(data);
-
-          var datatable = $('table').DataTable({
-            paging: false,
-            searching: true,
-          });
-        },
-        error: function() {
-          $('#loading').hide(0);
-        },
-      });
-    });
+    px.initSqlEditorView();
   });
 </script>
 {% endblock %}

panoramix/views.py

@@ -113,7 +113,7 @@ appbuilder.add_view_no_menu(MetricInlineView)
 
 class DatabaseView(PanoramixModelView, DeleteMixin):
     datamodel = SQLAInterface(models.Database)
-    list_columns = ['database_name', 'created_by', 'changed_on_']
+    list_columns = ['database_name', 'sql_link', 'created_by', 'changed_on_']
     add_columns = ['database_name', 'sqlalchemy_uri']
     search_exclude_columns = ('password',)
     edit_columns = add_columns
@@ -135,6 +135,7 @@ class DatabaseView(PanoramixModelView, DeleteMixin):
     def pre_update(self, db):
         self.pre_add(db)
 
+
 appbuilder.add_view(
     DatabaseView,
     "Databases",
@@ -562,25 +563,57 @@ class Panoramix(BaseView):
             database_id=database_id,
             db=mydb)
 
+    @has_access
+    @expose("/table/<table_id>/")
+    @utils.log_this
+    def table(self, table_id):
+        t = db.session.query(models.SqlaTable).filter_by(id=table_id).first()
+        return self.render_template(
+            "panoramix/ajah.html",
+            content=t.html)
+
+    @has_access
+    @expose("/select_star/<table_id>/")
+    @utils.log_this
+    def select_star(self, table_id):
+        t = db.session.query(models.SqlaTable).filter_by(id=table_id).first()
+        fields = ", ".join(
+            [c.column_name for c in t.columns if not c.expression] or "*")
+        s = "SELECT\n{fields}\nFROM {t.table_name}\nLIMIT 1000".format(**locals())
+        return self.render_template(
+            "panoramix/ajah.html",
+            content=s)
+
     @has_access
     @expose("/runsql/", methods=['POST', 'GET'])
     @utils.log_this
     def runsql(self):
         session = db.session()
+        limit = 1000
         data = json.loads(request.form.get('data'))
         sql = data.get('sql')
         database_id = data.get('database_id')
         mydb = session.query(models.Database).filter_by(id=database_id).first()
         content = ""
         if mydb:
-            print("SUPER!")
             from pandas import read_sql_query
+            from sqlalchemy import select, text
+            from sqlalchemy.sql.expression import TextAsFrom
             eng = mydb.get_sqla_engine()
+            if limit:
+                sql = sql.strip().strip(';')
+                qry = (
+                    select('*')
+                    .select_from(TextAsFrom(text(sql), ['*']).alias('inner_qry'))
+                    .limit(limit)
+                )
+                sql= str(qry.compile(eng, compile_kwargs={"literal_binds": True}))
             df = read_sql_query(sql=sql, con=eng)
             content = df.to_html(
-                classes="dataframe table table-striped table-bordered table-condensed")
-        else:
-            print("ELSE")
+                index=False,
+                classes=(
+                    "dataframe table table-striped table-bordered "
+                    "table-condensed sql_results"))
         session.commit()
         return content