From 7a7f61a2965e0f869328da3308fae92dab859365 Mon Sep 17 00:00:00 2001 From: yxjames Date: Mon, 27 Jun 2016 21:33:44 -0700 Subject: [PATCH] datetime format and database expression on column level (#652) * time format minor features added * add description for datetime format input * db version bug walkaround * removed unecessary comments and fixed minor bug * fixed code style * minor fix * fixed missing time format column in DruidDatasource * Update models.py Minor style fix * Revert "Update models.py" This reverts commit 6897c388e0fdbbaef4cee1015b1ea4941cf1ea43. * removed timestamp_format from druid and removed try catch in migration * Using spaces, not tabs * get the most updated migration and add the migration on the head of it * remove vscode setting file * use colunm based dttm_format * modify dttm_converter * modify datetime viz * added comments and documents * fixed some description and removed unnecessary import * fix migration head * minor style * minor style * deleted empty lines * delete print statement * add epoch converter * error fixed * fixed epoch parsing issue * delete unnecessary lines * fixed typo * fix minor error * fix styling issues * fix styling error * fixed typo * support epoch_ms and did some refactoring * fixed styling error * fixed styling error * add one more dataset to test dttm_format and db_expr * add more slices * styling * specified String() lenght --- caravel/bin/caravel | 3 + caravel/data/__init__.py | 83 ++++++++++++++++++- caravel/data/multiformat_time_series.json.gz | Bin 0 -> 38387 bytes caravel/migrations/versions/960c69cb1f5b_.py | 24 ++++++ caravel/models.py | 75 +++++++++++------ caravel/views.py | 25 +++++- caravel/viz.py | 21 ++++- 7 files changed, 203 insertions(+), 28 deletions(-) create mode 100644 caravel/data/multiformat_time_series.json.gz create mode 100644 caravel/migrations/versions/960c69cb1f5b_.py diff --git a/caravel/bin/caravel b/caravel/bin/caravel index 55911281d9..8c560f934c 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -97,6 +97,9 @@ def load_examples(load_test_data): print("Loading [Random long/lat data]") data.load_long_lat_data() + print("Loading [Multiformat time series]") + data.load_multiformat_time_series_data() + if load_test_data: print("Loading [Unicode test data]") data.load_unicode_test_data() diff --git a/caravel/data/__init__.py b/caravel/data/__init__.py index cf42399187..51b2914e4e 100644 --- a/caravel/data/__init__.py +++ b/caravel/data/__init__.py @@ -12,7 +12,7 @@ import datetime import random import pandas as pd -from sqlalchemy import String, DateTime, Date, Float +from sqlalchemy import String, DateTime, Date, Float, BigInteger from caravel import app, db, models, utils @@ -1020,3 +1020,84 @@ def load_long_lat_data(): params=get_slice_json(slice_data), ) merge_slice(slc) + + +def load_multiformat_time_series_data(): + + """Loading time series data from a zip file in the repo""" + with gzip.open(os.path.join(DATA_FOLDER, 'multiformat_time_series.json.gz')) as f: + pdf = pd.read_json(f) + pdf.ds = pd.to_datetime(pdf.ds, unit='s') + pdf.ds2 = pd.to_datetime(pdf.ds2, unit='s') + pdf.to_sql( + 'multiformat_time_series', + db.engine, + if_exists='replace', + chunksize=500, + dtype={ + "ds": Date, + 'ds2': DateTime, + "epoch_s": BigInteger, + "epoch_ms": BigInteger, + "string0": String(100), + "string1": String(100), + "string2": String(100), + "string3": String(100), + }, + index=False) + print("Done loading table!") + print("-" * 80) + print("Creating table [multiformat_time_series] reference") + obj = db.session.query(TBL).filter_by(table_name='multiformat_time_series').first() + if not obj: + obj = TBL(table_name='multiformat_time_series') + obj.main_dttm_col = 'ds' + obj.database = get_or_create_db(db.session) + obj.is_featured = False + dttm_and_expr_dict = { + 'ds': [None, None], + 'ds2': [None, None], + 'epoch_s': ['epoch_s', None], + 'epoch_ms': ['epoch_ms', None], + 'string2': ['%Y%m%d-%H%M%S', None], + 'string1': ['%Y-%m-%d^%H:%M:%S', None], + 'string0': ['%Y-%m-%d %H:%M:%S.%f', None], + 'string3': ['%Y/%m/%d%H:%M:%S.%f', None], + } + for col in obj.table_columns: + print(col.column_name) + dttm_and_expr = dttm_and_expr_dict[col.column_name] + col.python_date_format = dttm_and_expr[0] + col.dbatabase_expr = dttm_and_expr[1] + db.session.merge(obj) + db.session.commit() + obj.fetch_metadata() + tbl = obj + + print("Creating some slices") + i = 0 + for col in tbl.table_columns: + slice_data = { + "granularity_sqla": col.column_name, + "datasource_id": "8", + "datasource_name": "multiformat_time_series", + "datasource_type": "table", + "granularity": "day", + "row_limit": config.get("ROW_LIMIT"), + "since": "1 year ago", + "until": "now", + "where": "", + "viz_type": "cal_heatmap", + "domain_granularity": "month", + "subdomain_granularity": "day", + } + + slc = Slice( + slice_name="Calendar Heatmap multiformat" + str(i), + viz_type='cal_heatmap', + datasource_type='table', + table=tbl, + params=get_slice_json(slice_data), + ) + i += 1 + merge_slice(slc) diff --git a/caravel/data/multiformat_time_series.json.gz b/caravel/data/multiformat_time_series.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..e0877b707dcd49b77180fc7ea104976768aa95c5 GIT binary patch literal 38387 zcmYIwc|6qX`~HcjL^>haD`h>BEk?GICCR=N87HzvVhA&q9EC6=sV?mPBy|8PylPGTTwE9LM)#o^rHp6WsK;dJ2E#KFf@fy1S3{lm@~-t!LT z{)f$&v$j_P4mVzUpFWJrKioQyxlLEi#nv%?O`$5BbKKKUJsnMhjs(im!~C5 z>v~wl{bwI%wfGi%^%3}{TAO)(r`qUy5hgaU=1}-m$|dxpx#RHDBIex1;VQnY;+=<8 z%WgT>_9|C)xB7KBl?&MwiZJ$^-?p}c!+YJt8TKrk_2e&FVp| zoH3g}^s@azh!Vi<*?B*hQ)d;|=z5w(up=F=NW?4>FqZ=g1`df$+e)YuWQ~Q#be(W{mnDOLZ06ypyIt^F6Bsy7M%mr~J9eAssel{%NA{%?6CPu>Xkk5AT;B6)Hv*GbJOM3~jOk6zFk zN>BZQmRLkHLKXLNqHC_gaise!Ll^X)OvGKkG?PEzjVPhDzg90Q6?*pZhz}-}V1Z~W^Ggz`uGN?1&63jPB;qu*d`wc# zPX%Yeey@RqXjAj0hV*?i_&3pux)bQXfvtP>q6)dVw=1o!f zu>oSm8;TVZ?to3X&9@^LpKcL{1Wfinv8b$vgcQNtxK;Ky2~;bdJ2gGa%@I$+Y#g_U znigvQ)T+dcH0tX3%`^Si16O^sSn*y|v98hadsVu;I!0%`i@t?d53%4mHKfRJ`E|mn z7}D2)mD@hHS<}|$O)x&|+Xo=*qe$!us-yvoyPmRWZFLCi7B#8rHfi9jMv5^bbjn6b z+MRNQKKHd*Q?^DALp-MS2$)cQbEYI~icU5{F?RX}g=tEjqBDa)r@!>Rx&}*D+i%O9 zEKd7HvZQRd)a_=3)Xf%{Yq}HQ32D^G?u$rCIM-{=qK+OEi{V>y`&khCZkazkm})a_ zTxdNP^&9bgU%kPa7LKKT?&k%!c^`3dtAHRDLkv>_y<@FGlp_=pYgv{2mY1)?_wmYdnw?(*sgIsotKJlCV~B}L zY)Eg$(2+XN;hq*V^LY>P{vr9-iaV7%{X*(2Wh55TU~ydhFcO5;&mUj(LL{ zdr%uJYdn;7>wbe9Z_sT+HRzJ4w!Kjm8`6sT%k_B;u3>EUj@JOA0CEo*w3fjV>HU@U zjfcnOAK5EKHx#g~UyRh*3s|JV;o8PV-YnXB_m`xI(;1>(R{0!n!GrZJEFFnN=4v*01@knKPkpHJFc!SPtT)$D7p|J!`aM+x_VpnyvRnWZ&KR?o?E zcj)xiQc|%%E0kAuqt6?VozjJ6MhzQPBIgY-q4t<4Zg4NqW=0M)`+77WTcF^-uQl%} z$d&#w@(y@p7>DZJ$MYwYa6#aawJB>@_0J`#K-qQMU!TwSmY=iJO5 zN{x6pe>aG((tjVv8>twv5jg7IM*hB(5`2ltF&H|I+l;7ZSC8)@VQc8xHg~H)Kd>Sy zyMkMS`8t`UQQr^$k~6`)5d8gmd2O&6gE}8GCVsB}6RFI{Ew()zNtyd^Jr;CxxNAbc z7zVdovuyLnKpipD0q^&El$|;D!R}2zj1Nh*ZVQ})&IlBU{Bf7!^to|H-|g7pO?Io@ zXB*M&zn>oarQB(^*Az;PsU^%Fe0xYqUY|jU!6+0_e|#^4@PzT3IErW>-hzEWdAcYr zXrAn5qu!eDdZBz!y61nH`$j*v3;lZxYFxsWyeb|G}-#z zb%gnB+NJ9HWZNlLyt|$pX(>r@XFGY+b}EdMxU-LTpfemf)! zp46lrcnuDZ!^f9HbYl1X#x|wo*MFVp;|&?_{b}sJ4r(T}0lORLrEiXG`MwZP0ca`C zs1GW#8&HwuD7b*j*nY(wkY42d;O&!xy-M3h_Ps=%ptWjT^j|BFODdJG(Qmox3tCQz zFfBSgx#vfRal2l5gWYgp;p)WSS9b9MP!xf8g)2k5Tk=;-?6th zeW>SGS>AVt{*~%e1Ijp*GrcmquBG%$1yJb%`lA)0qA{zT42p&cO0hOEeo@)(-k9O} z{)ZH+^(L%y7X@2bQ`jWPf$tJtaQDeA^m5-xqf_GQm8Y-wd53 zAH{XTtJxOs06zxz>G0d(O25D_lTrw98($H%XX`TADdWMY6<5sR6jUc4fjclCA-ARS z9s#{FvEUey&wb!>GRMqeZ$jx@*7NY%#8_$9wh3PxZ=I%&w3@N+q=Om#BQ*9+yA4b)SM)wg-;~U*7CO zHwarYsekfBS{^?4nXB~&LO6dZO};C4W{>Q2sq^ut{^|WB=e;uAYq=Bb44Yp+&s+}@ z%Me_Lho`Z4Xf`itc?8bouv|mdpn}$7d);@VAB~jWst z>SunyvM@qAbf*MJo<|#;5lkkyub~l4KGD6myd}};)R-U;qfrk#ll??OX4GQ0e5SWt z6&*ZvAv|BBf#!qVZbFGkE6T#}psm1heChof1a%%Fv59q<&w0yrhf&$C=9MkBA)U5< ztnntxPMQhD_Y4GQmw4_NP9_m*|2<`*Xr#@{_SN%>v_sBvrk~?hIH4z6mSnD_!`{nh zUeL3AtZq*GNfxNmYFd{;RjV2if7j+<#yb$$N}klEWZbBr#Y55N1} zoKc^a^~%(P8|=8NcxE0Qd$La21Ix3D=O*pQI@=&!>D)x!5;ho>mNVnjFJs<7gl%M? zpY&Rmy`8EVmA+#52|gmtB2CO0k?fq-cwS7f`v?EHfzxjA?J zpfw&-=18Kn#FCfm%ZYS!PzHwx(;W_aEpZ`(!S=VppM(T6f*0&UB4b3Kc}22ILeWf^ zx6{^Y>wTjwHK~DTp3@d#@-b&x&}Pe7C%ktqXd2>au{~eho%t|1AAcU7Ms)@e%JXfj zP1-52nvlQpk-&3sY8$|CBn^g>JRfZn`(-5DjZOv&2l^;jOoji4ECi>iEH@<nG@yta=V!CN`M^UWv?x5 zu)RzVlVi3Na#t~$$(9xP40;dR`bX<&U)3ZjlRg%)-5IaivukTCW4 zQ$wHB`Ah%Ra7!}o^lg9lds)t_f4DohDV+Ik_lmIuQ<53@t^ z1`GtFI8LzQ(_oHWQ8U*U8S{d;v)jqTjC=X8cem>03@q3SVAr7*hl)^%@F|`1U3GWcXl%zlk#}^&&|vl$DoS~ovCK1ceI#lfpV64$hl>eDqbZT zHX1iZ+VNhxb0fLQ4b@ik_7c;gXr1)woHbsi%Eo0fevS91^HYKS=<`q>CD49KR|Qdo zZE!6uEjrL<0|0lG8mXUa(h%!Zk{vIj9aYw$9z_9f_75Jrr}a}hEUxz7>bNePM8Dbn zmE(ClxvX9#jP1@DuCAFbze{G8dwAsE&(!$ie?Jw_?<|B}g!|zOK@;${Xs8sx2w1_t z)Ao17+X@7?t0g<#vyS#Q*0)h^=MvK{`|{;A=UMF#kyL7N|DrLzm^Jh;%fzD=Vy2v?i?7(Bw}IDH~UI(s<3`lBje-f~f}ghC(n-pB z)hvWgsK*ea9Q;ShDr_+QLNB_W6cf~Dd)_!r@gb4y64?*yn(aw{DKJ-y5;mBQk3d~qe~Wah5}(RF2Me491r_S{EVEaprQicr04dtH zKi8&ZWz5nji?9F_#-#y&tF0!52j}(1z@``l3ckLGwvQVL$RrbrJf_ zb7utj?9bEC5;bWRdnOs~8Gmf=g&sv&1Io=P#h{UM0s0RVDzeFqg6fdI*v@z+u-ZC}cCU5j_LhEg?XdBOF1U-v}w;pPYvgov+1 zCHx;EUbb}m{l3J7Yrcf9qG~UmwA=$>pQr5xDXuS)&{%Wsx?$-thtabPAc2GyQJULt z#EF4WLIp~WL?v$wjEBGXr7AP*y`Un= zDH|IWYNfaYi#~2S1J|A9+@Myi7dy2045))|{z44cOn~ZJOBfA}O0OS6?Jf%&H0wW= z9B!GdAmU7hV%I_m)w6~ahAz}z%>0PQd8Fi|-CoSX$X5|F=yD+kpQ;>dwz4p-XFya= z{f@2k*+B{*iDbZ9Am(S~QqQTBPvE&F6>n88%iIY*3_muQ8aXILl5qCBq~IGstzBP{9K?;t}B&(50*t!2u~ zp6J}}d;)sd3#_L0(ogqdTRW_aE}>O$-lYHpF{yA`yw!{g_I!FqeIk`Srj_qXLs`_M z_cLk{dYb{kVG~`VPtSyiJlA`Oc%a4+B*u=Fb4FJ-=xWEDQ_`dMp00vm&GtUB`Lz0) zoySjn>Pt>O``ZAv)(_nScd;>mQHe2gd+{dfD-Kg#%G*6hOAE^ekubvVA_V$gj`#aXft16`U`?SOuds?Tr?rbJNS}^Nu2$vxyM-CZG zVq;8VX3QI)3?>BGOc5&&i+}yvhT$)uL;bL#ElJ)366P$O5~;% z=#-$sw@1*%j3#Mu#T~1et)7C>WoZQQuK?6r>Pr$+|Z8w!OB%Z3`OJv@rw@6T#0xxSNa zQRZY{v5cd~-f#>{_sT?#Ow#3z9Ija42F{coju|R#v+_C7I=6g5FR}=>rYHIjN#{dK zf0sd&g>w5Y@69w6SKb5K6QR1EWo`EsK^e1u5=WR>T7r+g$X>M~@taeVzw zGBIc_v51V_Qg`zMXJBoEKpNPswVVnYGQP!PxIT2V8^n%ZrdlWREuziKwXjOJRJa*l z#0qV#Z~oJaEvFZ*lnu0x;gIHvF*`#op=G82lnKR&$9uoxqzg-m|JEgzZ$y;hTm%)A zBw4@BUs3u2Q3v#J$E|W!>u|u+mb0dSqSW+QbkKhDUFPQ8KwP$8$5)Vo0KFs6_lBSx z7as3)+9ypg`bJp#1;ukt!E8Ah=-eTjj-aXxwCrg3P7UAx8f}y4wm{o+Y>IzKW}uZu z?}giFaQK#S(wmldoV6{`yyj>ZHTu;8w-?ya_$eI{Dux z_ysS1fRVtdf;iSPBM+jSPTd%di~kLpowu0xZqMnwq9V^Ze-+k_4WxJ48Ze!6Ou2>} zSPojlhU#+%IHIy7rtJ7to^Dxy00x_siq()V{N}8yRpk)fGtRR!e0qmg?iMI`3Zcha zQe7(psMXdZ9c^a%{A{${fn%NA|MPF{@VTCx121=bED7GPs zw)sAigGADq7EFB(Xuj&`l`>ORwsO@#ZVc*9p{&zcW7AmBRMB0awg+IJm+SvZ5muwm zGQ8>d-AFZ-h|yl<7X$x6ne#zR+bNH?W39Aq*MzCP-jlf_b=lw&C`QF|d)H%{f?bzH zH~JDL_Fx~M^+9Zcx4trND*_=_JSQ74YD~_u4WmP~;XThAXoxG8ACp@6POM9d;<-8a zn433);{(4n*JRCL`zsZAunB$^V2*US(0a-xrW$F8By$SRjhy}|f38$tU`Lag6b~ZV zF&MB+f~LiGHq5sigF*IBeJ>H5ULRpH9GjqqDH&J3TK91uCF*tEXJ$tWKB^V`GR})Lv~+*EHPHPy=~>` zL6_?WcVCVe7ep63> z9V^turBI6tCp-XrLZNIgD)IJs_41_vOQ$I+yG9U3C_rD>z}H+_E@5c~Htjp*JvQ&7 z$El13?Y^(MBUYhtWE0Q>zpT=Hr$o@S3tLW2s_d2G(T9o=WP&DcHz})x7C?9u&p2Mf zGdhcYYl-aYM@<)bPQ3wPXTV+;%I~sdS4VMeL^%t%Cuq1io=IHGhCNrO8@ls|arri4 z86u+`&!~V~l}X4=FHXN2;2?csz5?DY6AX|!U^cL=x(|Qnl4XL=O3EZO{7*_7K#kb8 zfSOw^Q0uLoZTfSjBr0zS%$ppu|=`xbD1x_kl3|x1MF+tls`7 z$Hhz){!c{HbVhx6G$!X)ACA#H1c7YEybZ415k`igzxR8mDj=>eGieL zSK&=M;af-!cmRGHEZZMHZK0+tUh<02{@6^&%XLcWUDU$_c^V-m*y!D#JrsSM${l2&|38c6)%H zqcio%i%+oC)HpG~lZHi@^V66rL4ydJp@4XX%gwsP@{=aS5hiwq%Lbqn2pMQ`_;=(^ zMR;m);7==Y{qhu4hH_tx7Kf9EN1<fUqYh(6UOH;X~T3~J<(g3 zQ?KXQ`yOf`k4`E)!62vXGDGi2Jz}Fc8xVIJee7;mf*@nz4!k8-lj=Q`+7v=)UYyf7 z5(p9%1TLKd>DzATBGw1M4~$S30#Xz-t%ets{KUkw39Ht=fCxJ}r-;+~l50Rd-T8$o zv-=8I$PPYxA&@kPt_$Pj}?JvoQ4t^vN)0 z+{yzAipfVxW|pi{0BE_i4st*9z=ydIjj^`~3wzRGi+_u%Q2#hB_U&+MnL{guZLOY6 zlEwX$yHF?OLWRv?f}C$R_sGtXN@e!~NYw12YVTg+_~|5Tmbx{*gx}+*fsjX$sWT6! z;|@sK#-Ad?J**?G`(~@(E3nq}iZ32a!33Q8pni0LVZ$>sPr%|AHY#)FBj-Q~L^5jto~F)&2nth+sP;bQ6?kR7 z7`YhryW;yyiE7_{$WK6l6_`}&jZ^Fbue^@ikB0P|E~n|ciflRJuA>#>OS$5|`G3zC zlAa~7LRJrOSTcWl;X-(;2eVg-`(hWqmx64p*~OEjHs1U;q@CL;SYb`xn+HLC5H+9>-1S=D495CdBxY8 z4P8`4N#9&e4)}EBftbK}=-YFaJj!iy6UU>ZGr(b~7G66(kKT&6Dwp%a{u`I;yzvQ2 z3IKHF|5{_N<;@CV^?CTG0AP$$e@)122^mo4J_nUZ1xHb3O%iXDRrj>R2_wdhzN5y8 z>m%u$0axn7m9y>^iJ-sWWaqr-yh_dUMXBnlC&XYmTzng!rhw^(}nXECjEA~%p{O$QYJg;L^Wib6^_rLnPvc%u( zXYY|)KigGgC^R0cB`@#Ojf@F zF)Zth5mkmNRc=G;M*y4R(5R+yrJ8Ws2UZl3k1G}VyZyOCBf|gza~>(3jxP6pE#&!o zXJzAviNK4?rU{y_Y|z1x=03MW&o{0XF26d%I1e*OHgOf_jLFkZuSU7>BW`ix(^vqp zd}Fr7R@46$^${QzU z5uw$b8n7`uTh6|AgxGwCw!3hkTl%wfl5-I)4alpW6Mm2CZDwBi7Io(U&fd5z7+vR0 zw=u+{FgS65RQ95x$&UhedMvGrrU3K#71!X>y1HQM18aO+jdFshPlUH5<2?U+yzyzd zg@x%Z4y+h&REhQxS2r`9-Nm8u)&Kmdsn-spYwR)-=wHK|N{w%Wkor$~IKMxZ=Cc~l ze8`7_E3f~+igoQq+%dg!w}bh1O-*zK`cAmbxb`EB_s{JO+y>M-eC*C?rJkE+-ijK- z95ol5veDjGjT<(`Pv?PjA^XLf0i+8@8%o>pd1XG<1etleZ^oFrbIQ#%fY)BiovKMO z&THdkbmBneEh9%#FQL8kD#bnoRTWAY8205BMo3>7*NW{@k4mI}PFNVxR`t4k@5D-R zVT8<;Qjkjl!>-uU_Xak7C-Tfw{f4jK`zy-;u?INA0IKX*tk0^Az8p~DKL53{ZfYq7 zTB5wXj1#f;g$H zaP>Wu$KpV;{l!tX+y&e>=v%;}1!Xj-1m1 zXgdB9D81xiHU}D+cU-d)R#Ca%X9SIEXqnM6l53h;_LpGeOHP=5%tScIi!vmtUTUI` z9Q$XR&VmZtsYdtB11Y0DT35%Rv8-;8H=UB)gAQl3eWAW8FNJn0-hH68jO` z%+f1}>`B8TVbU8&n zMwSTn`AUyxHY1plsE!QEC)Ft&ALeA?n&MCuMwIPQ$*3VkFfO(e0yd9(l2^GZ#huKe2UMm1iG=sih#t4vTg_bWi3h{}poR4d zCU?CvNxx8g%iBTwoI&>{Q;>7ir|P(Lj^J0>T71>Kf6-i)1U!4e1y)Bkt22K{+M^6Z zuAqT&PD>nEdvZ?~P@ilu?k`q1MhoZ$b)Kc(fZPwzvi=QEh|8kD*555C6+ zu|F(kOTKJ=?HJ?m-h?UrC+o-F^ln^IIRX4U7G`8L@RV(Nfg{kz+gEWcU%p!3ylwbQ z%q?V+@IenxV(?6j(-|e3Z@d0jg{kSc!JDA{l0DHjj8)@Mz5m9BY%eTAHc2|P*>9-< z%pYKI=J8{>K96o)USnGRNI-duX&S0g@HaX1Q#V~#%O7uZQhgEcOOuoHRr!zvK*e>q zI{Ow9qGXEgkmRh9GTkkC18+&`f4a&#GUZ*kI)UFw{i5|vzMtc3h~G{=15NcLy{;B{F63~cJrWPt1JKgBUtLhd>4i9g|Jh*7M_~cT*RqB0rG6L z0QYtw^)h^hx&ycs$1}+&`2AF}`~znuLfh;`e0f!yOd!x1{}c23@2DRBG<@9t2Yi#~ z6G;6ExHn*_1`hD7+re`Mm@}3_XH9Qk6ux3pQVmT=)Q{$LwU-Bhb(1oB?0=DPhKXdMkcXuVD0Vr#8>=L8Hh^Sj? zq#ozLY#`EhiB2v^>>BZD`{3L5r` z5jCT0X)&i@_z2-&7?wgy!vzJ;{jfP`pHbs_w^EKs(whvCog^r0XTc049mO*QJ91M8 z0!W{zz(falgF1?+KEr)>j}d^GIF*aY5YOPFzm}BWqPBspIxi~TkJP8XFBP7`XMdEJ zCaAJK>?I;MI{-ma!vyxyN1t9_D%>oTOXt$uV9w>0MXOUp*L2BWd)OnUsfaL|Q~8s#bg#Zh(VVoEZ)=Ew%9-@1)=nNn}TZSciC21}~y zdof1enKM}}?rQlw=5R0izWm(E0x;kRi1uS6i|*U^JJo#p>I$J3K|2B^u_Zp~qWbQi+3QJ=gd7GElgj3c~AUT0rW!)K+hu1cIe%V`tXg8F)E_3Ht zXAp3P05p=j9`kYdo^Z!bf4~p0y)>z|=y%^rn*i_q03X zZ?u%)1UnVN=JJKtGMjl$Db)QbqMv-|uUjl!(&YB1NO$s~A$ypbFL_Tjs7iRPLO$P} z+Vqg#iz=_8g%*bP*DczHEW!^uaGzoIKANA>>jgXOb86G=;F{{k?RO544&C{-8hbPp zLeUA8YEdrQMEiVq6VG~42ey`G&^PgNefG$^Qb0)ieRCoq+m`>}$j-N|ouGSHt5j_i z#!wS~ac&pzi3@m$pzwY}!D^~la@XPjUF9b|w*?dx^@bt92TT|X=e%nQ1T0|$r`1$0 z8#|hgyOyM9z?MTQ1$&i2(MgaqXz1CUtzM-;JF_JXIRoXeTJbQ>K{(XIzO17b0xK}) zzC1g}e42wlUN*Bqj=c7w@N4oBn7yyX9gy+kB($cMY(W+QNHzNsq6#D@Zch3H{n*Gm zS97p8VFKnmN0QUL_L~wX#|y+lL8qEo=LG;=7(A~)MwjA#9)DR)4L@+G{JLiVv3CR% zb{OgB@S;M|y~nmXxFoC1M#X%PJM`StO0O_gSirO5S0Hzuct7kWhMc*t zJm@nH`Gw(J| zdydi>Ke3>LWAw@PZ~NfM>(!+;~^A)tS)GV*C1f_5{7@WB&;zP3Zo z(l%3X@ZuPc<*{MlE%^`mMjOvG_x`k$Vi+9G5UgbnQaOSNxZlew8(g^K9D7-)x+r1d~{{qBEUZ(sDR4|h?_ z`>Fwai$Lb5nMiX#&yeXCYh7J#0mBS$J=1hUk?xYr>tUEb{Y?5zP@g42NI z863Z<`)$S1(#jPemdF`~BnOnl(aEZg+l4V}7%RLiDp~`kAhtoA(5!TJM(`_Yeu}E!p}Fr|I1HL_{c@G zu;ijsB?io9Zq{xo%8FlsX?JrtBRHsv=k7HrZUSa~-~e~<%L?#dpuMu1v)a`sBFjNN zS=9VidT7D^U@mXpAr*V$XFesQ`&ZYK>_yKBS`CGopKjWZB`r^W_$0J;4A1><$|Kp> z`lj8zQIh9Ra}0_lxupMSElWx7y-X48eD;4xc8N~AWQa7uvXYig4Pb7)9`Q{4;5oI0 zY`&`L)^7*~NgzSMl+hQR@>bjtn$r;rR_Zn#0PQLSc*&&eo})iH=I{ut&xIHNy6e;C z$AGAa71lfU57g=blD~VhKmEd&G89&U9us0+<2IN?xF<7JIWcM8Tm}@LEl7_Wtlwv} zYX&lO7y1`jerx!PE0`EFF>Fk13AQ6U9mH|Bd`VeSiuV0aA(R=9CL%Cw33BlD8V^XJ zK+((_ktR7o@xKtWW8SZ8#L4ZX&$BFicM$;;xW4$+!CSN1+;!u4tT`I;+xTX&5lx~*^nDHQjzW%CVHuPYL0xF|h% z_|a1%IWAoTC}po-f_B6V`uX&u6gZ!IdZOpuof9VY;VSk6uUR#M5c8F%CywV;m5m^& zgGoJx2OHxjs+uEB0mTfN6z4~j2H!*mE?G-XYFK@wKxm2z9Yww%?mX8UW=O5nmXhRM ztxO-Eu8f9gNM+WQ~$wb*~;&+YQI)MrP&Ap0D# z2?W10F&jEd7pOd^mcId;S>gR~n=M|tp;9S>B~cC{^ZkyIejS;uoc%7XuQt6rs63b( zzf2+^wC&x%ia7a8>e&)XMM9eYTE1h?IIk4S-K_t%tRtE9PW0*)xD${x!70R&)HuG} z@&wkm4nzysCBA~cX7UXo4rF+I-z+ ziet(yE_eq( z;Q4O(K6SS_ZS-0p^iy&R!=O*D;JBIPUR5-t6BhjHztUy1vplL?6tSunsj{cx=Cp+d z6JtExPfy?40<{5!U#+Y6Zx2uWP44mg8^QC8iq3EHBh-7n@B&U(HLZ#zhsHf z23@UXB0UR6y!8i4StFHK1JQ+~&?qaQ4x-g6uWW$PP}Dq(4j3!3$Vj#OppP-p>Cn73 zJ1&=F@Jt^@o%NZ89$o;2pvgb0p!;?F+=-e+ZWGA=M7=@YPz=h`^Q#dge*BWoLH+IN zTz+6Ed1PIvj3&3v$zuGY%>D@YnD3*nr)9VvAuEj2Pc6h2vUkWVjEHc?2vG;4s>+VY zdx19|w{m;G=0G$!wid_o<1n)@0*D>p*zU&>Y6VClb2{`&sc{z;NFSy}Jz*qUpHYiTZGa4z~&#WVF6z z;j=(pP1t8N|0K~=6c9|a)O@maC7PfEg1Ozk13P8&10MRphyf#N*X;4e1oMIRP-%R3 zC0H`lJxZ*hT@rLojqFBbu3QGMxgdSM13<<3`MHr+Ne4gzq#GG??<_`1t=xr}U@$x& zlJ~ApAs40lh-kOr37F=W1fB{=TOL9Gcq+T&(j9QdGin16kw@*Uv-sAkwsL#G8K$CL z*ahGZUM|z;H1O^C`BNS#U0&T6JA+)0!gkOmd90(jOh*`1RemUc;9CCJ{8rqR#jxJC z2~!3+a*=D*B{r@u70`g44JgP*c3c)eM1_zI7?vQz$?}fSG%=Ouky>)~l+J0*2@Iep zqxFUSJD4qK<&`?d(S}3-^@gsd!^)jjT_0%_zeGOMc$@jm!dlWxE$heFb_N`f>O-fK z%1^R)GIO6W9spceSHVf}c0oD~89?0IX`kozcKs>sZnkW<m-m z59TPL#;%rNfy~!bF#^Pf|4s+wL?{Beq83hW{c8=lI%)|(nebNWgB(LL3H{eQ-x55% zg%H_WtLAQ18nj^nnS|Eq4|Z;f_3gSdz_JB2%HRBaY$w-7S{^>xL9tE$gG6VhfK*Mh zac;n_1pIC;s@O#xlO&aX@y~-a&8bH{r%XsON*;spRp(%C@6r!tr6&Oc3C&(?E8|{g z91@Gi&6&WIX)M?NP{ zkr2QF|6_B}>^o;Z+}`xNIq*B-rD@+0uw99wP-8B$jfw~RX!{?5r zFQwU?9_H3H*ucl|VVp8&l!i*34A*raOJ?74Xq%$TwS1n|Eu`RmN(4EBrxjEHzs_x6 zJHOOc&&(@JKu&f;h|}q3LvhdmAfFdrcYdI5pb90Z!9nlUtJ>Z6XuWAXALh0p&2Gw^ z`{rT7OHSw|7@s0X@PdJg@y9}P4$~|<54qTfcRhhEbq=Cj1LRH=GNXyx?EYnPz}COw zlHv?V$qI20mte@dG%YZZ?FwL6#9~gYN(lz^@mPZ{c{^HRPT@RE5Keq8(7G1m3N%iw zRVBNRKv#_TCswmNzq+sh@qF9%d@D`Q!xu()lv%NtX#XL(FrwgK8BS;R5jbg>!I1uv z1cR-f`;3acY~$NL#z;&VaLg^sk<1(Pm41zx_X0%pF(l=uGZ20o^j%8tr*2ZK9^}c$ zB#3&wONSj~lmQ(Y>_X?ny@1S>sGNd7{$uDGpkDyXrNlr?%g_Fn2VeW1`E zIb{qblide*#-omdLDx5^xl|zkf5!tNVVkl2`paekqGq~<#=Hsdj#gYSE#iP+<#B7= zz(Zrtg!tO+`g5EOxMv>bvE29|MS)h5w5-$uCbyh3hOS7f^Dzy&#zuXI_dM=jOr)RG zQ0ZG@7)-AhH^yWKr8-(^{Rhl6iHgt(81A7yyAl>hipk&r^TJ6UU;hDP zIA+H4${(X`n92=1y$;aq`ZSGA**>VYc&7L*`_r2EC5=jPU}*24BNghVVuxldN_OJM z*G6Z@s5 ze5VvBE5>D1tOciPq-Y-K*J z*_=_K&Wy*a{kKqAb>wlM;qElR-_$!3pCz2mkuGXlh{X?oNLXd0&(ieUcmh8{pCi&% z{C78=FxJHDYK`L!*gua*dsB7n^=C{_Wi{fwK~qY8isKNS&cj_h1ITd&*1$1sXkhct zpsKayQW4GL^IZ)NKpPBsc!4OohAx>CvD!@Xm7^szg6xSdU}?68JAtVkvWubwIt>Q= z0L%lW=&U>W^z&Bv543~cl4|}vYT5B-(V91=D@ZJt;XlM;LH_>qwIos&C^-GL?X4gA zOKz|9cgPKqx(;bX*aY>f!Jce?JXRSutQOt+|qT*)9t7_to zFB&ii5KU;FZLs&mB{klL(P#Lv5RKdtRkB0TO^@F!L_YJj8OKKydn%NZl+tWzCbEg@ z^5`H;$>))?fS`yd@q~yx9)~&Ii=Hg24G+E*dB0%szyMadU`pL>F?Xhh;v|5!)I}9s zp7I*HsLuXWf4B6d;x(e>{Sfczt_JVO5|;jlJW1;zH*#PU->pcoEu(WIsZH%lepDcu zPD#raxzd@=m_~G>Y!5_-+F#v)`K&As2OMdeacSHl&JuMK;QIqMq#-)GUWaE0VCX7T zvzF`tTgPy%Fq{MJ?q~WR3t3(rS1Zl=q?B`V`g{H=`1@q|Jg^V7$;XfOR7B*8Csp6B zm6)#xz*ZI0C=Chk*UFZK=v;9ysU{v-&Fg4bUp=L;=1eyj@jK+48FPL?@l`z)KgCHc zmcai_WJJ3p=-$tv^4uks^N1GH?Aq**UE8(&_hZ*l|Bes|*e1R#%fp-fKk(*RVZY%BU zL83MBx-|>>cZ2?4Q&%1i<@Ev!h z@NXjf!7wuun1`Eqm%d8B@3A;ueG%;tU5E{Wv4hZ^qlF_k3Co?TxNc^yXdp{b3=G@o=1j9WM>DxV0)Quq#ThFM5 zlU0qVBo&CWa@028@J>_-wE-0{rU>9G!AMUlYF?fo?HJOxZN?4H->7AjU+4lBDK$VJ z4Bt0!n4tUX!W$T|jb8$neMT*L6rPAJtT`7T#$3=E>3TSx)Km8eI1fOii@xeuq$^Hp z5R<7148F5LU~-wrGbFS^)9s$ahW62Re6Xc5w$qh^^;LG5#bejz*%o}};pR0IBND?; zaQd}pPkJ2XU|@VnL#!rSKRU&k+FaIEfh8W=gO3xGlD)R%0&aP{zYt#3eTEByq!V6% zJs=?Fz78I@@g*56M=ws!OUdk$2^=t_7pWQev;Dmih~%gF!6k!c^ij|3&!6%vsvA1H z5^msGM5!Go!g~Q4MgwC}N!!cBr_|8zpVTIfJ-#_RWQjTMqT%sprw~A$gJDGzUt{(+ zz;r+sW@0pAOR2Plg5LYX%0Ku!_g#imVhn%9-upD#%!gHdZegV!jjv}P^1E6w1I-|V zdsI5&9h2V$!7PBGxi{ukvwyQ8lYOBOSZ&Pb;kOEGCg`bJ`@HnS3AJvHFl!ti2Xy4Nz zrOzHkQ_7_+ZSHa5L0bs%qWaEO-3EbQn9INk_oa%ig%9X2oULSWuE#ero0F;~FcuF| zjN2VeaKVsB4@@nA={`lZqH`oq4kX+-=~)0L=ufGvSWrv&9lk&8)%ot!d_%@_M#vbB z+H)tzNDx-3`upUU68Go9AdC&%ZYb4a65Qicc$AZ-ih29ceH?uYc0KT!dHWZ#FF3=8 zyCn(Xyadgv`tReQY$G`FIq&&zruH@3LOHCq^XO@;afPY2HA92~n2eIrHTb)@{EY4%iXrvPow!86 z5)$^A=GRJGm7+_LzOJNdLrU&DdtfTp=HpJbc9!8}srjo*+kEZ-_ZW#CXxlf}<0-{_ z-0x5`(+o&IEEA)S9{1?}SF4Cp_L)*|cx{*|1l{q3XkQkf?Tv}1q}&*~ziWZEgDF*k z$|k1|VW}e_JdKXy$9XVgfka@*qilfqhU|gi4(%x#$YYQA(AgvU?^H0cGR@@NhP@rC z)0M*GTpdqU#+9eByAWz{_};Al&L_QzV&gn4wav$4E6#$yXU(jXJW%p#O+$y3-3cs2 z1MgGuE$Sd={RjVy%5yrgPR)e|04DIHb^G8FcxuSDuKa*vGiJcbBwnl8vB9*dqFn5E(O*PY(!@_3ZO~KyAO-1IfnV z7Q=?C03?gL@bX{VnE~b!h%xUAu9oinuA(f+@zcR8*+}({_RnAx6lJiAsnb0hD&&$v z=%8&351dYe9mO~81690Z&*rIN9q3HYOL zC){*wW2W{ZWnV@<8av3ATz*Ulhj2A~=F}~VuYSDnnZzsq>EK>0+yG?PKRe-bh6{}Z zZT1z@JJjZQV0PirZFP@RlygZb9tr%>Ea&pJ6kZnm1DN!oO;GsjPIyt(o%>F6*xyZQ zu)m&$0mlcPjH(aPbPwTQv$+=<7~STn@qA*x{m3YnTqtrXC04rM^4h|V2?Al`X9n!? z146X{5PTQhZj_@3#9MJ|D^w-?Y->EQd=3H2g5s%ygjzL8>jlD*V z#75Oi2@mKQ9g-ViS?edtf}@I#fODCIqqd!sX{x|TahXK!)GM%%Vjclkf(xlu{Zder zW}j7O2CjHEwj=k5SgRzcd_sDDWaW5uVSiZ;{kjv-q=T-V^Cy5>RB!GFrFPYHy2k3G zhi{e^*o+~7=$6rF=4T#yo|i%}U04npFj=+y8vZm3s1G|>6;}9pav35Q2!-Q8dw&$E zva!Bo9`SlXCv!Oukqgxd@<7Uvuh5+KU8O`Wm$fW${c3FfEDJvOVas+8m5eg1whxv! zd#uvhHxA0gu!3HIWVHG`b6MB$Tmh93|>;SeFZT=NY1; z+5veB8M9M(CT%GS>oT1zvRt(I{!S5Zy(KrSU}#%bm#JN>!FXP^d_A%deCwo;E$}$f zR~9^7Vk^Xky>vnAHIHTjx!ylT>{KgO3Us$)5*&POtw1`|9&bF7HZiB ze^{y=M`{V(2Q8he`6+;bR!KYU2#wF^w?@95#I81ZKk{^GZ)aJKL|peINnQ3@8{O@P zHlw_ULK$4->^$67%yi6Q-@0`ga=d~$SMLT+iik%tI&?#HimKrTj!tJoIBbbE&fu3S zoWEswWg6P#Xt2qLyXv*yj*x!PjVVvlc0z*Rp2jKDsE~!$M2h(=LzLS|b5!Z2_=BeL;5 zxna$`zM~b)jDh4Dnm1%^(u#__Cv-s$$VfpY5#I-pqs^F@1`GhtUCYK2_t+XU-^sY^ zvyeT6(^6q&uN_F)$BmQPWuG+8I@4s!bgiLs*X!n-1IRu{}a#?<9EsiVA#LRKLa>J^9NuD z*Gphtx)|k|SB6R{45k+E%qlJ6b?ZsM_;D14^W*jPYe~Qi*4^|sAJ`kdcgkDBMIN9F zT^x7_Cy04_R_pJOtV84!fd-2hr&90( zkwc$=fyHuJSzfy69sJp+w3r#GHJhVe-g=9dv4-*OyPOp#DlWJX&;nel80WK~)8WiF z7N(cT+bke(3_5LpA7CLTd=7Orf-f2M*hY%wvhttBuz)11)*yR29YS~{Y-79t8MW?! zn$XgiW~C-Om>k>~LegXX7V}iW0l3Qk1v&~~N)XCHU+wA(Yjzt$=@9?MJGX9V8;|8hizoP%X%hJ&KyuDWY?8jun!^FW)@3X;-(Vexy#c@r~L&{_x7w^Cqjk zRQgcwH6S(^xD0}zx?!5(^3hf5QeVe2fO4Q%FI|!doBY7gUq`p>l!8{Nv*gN$kiif> z<;100Yft>*<8(r5!>MWYL1l8(-R?%w(?V^7AKwaqW^hR?Q~ZA_;!di%$0MauuutyD zkSRru#h@G@tc5~qnogRx_|(2$0=;~Ial4Uo`@g7^^yskxE4ou&_^5o8RkrC)WFN@c z0n>-!bd9;wb9|HzKvGnxYwq;;!T*N3{P+Ew-~mqyuM!Y|}6yJI9r#fHTN=bU;7RAGGugB1S;Z*Y}zA3~djSj8F~Ta^8JeOc7)rIt%Kx z3(woLbu0f_E{8NlcqOcN4B3)f|8jG+PK-K5sV5L-3-iu`=$W7L_CRTxxcPeXsqV%7 zkR^4aM)#Jbs^EedC_?}gR`uSV9~St`0>JuPHyHr;o-5fb50w6A@=VQU%{FMF1^Z3#Rq-vPes^`Ut^1!^jOeDOnunEdCwogw@ZZZc?kAXkRs zoW%SzD&>Nkq4PN6(=4Q=6ZLBbb8j0sVs@NUa)EEZ72u9FVBUfzZbdjzX4DKlcjbj; zQ@Az%4Xo(`pXSLp#f2250e|xZi|DDJVx}Nvbe`NKpZDsG$b+xq_Z2nQatfKXZc!F zSeZT0$&wF>KL;D|9tkmcYDC3>o&j*HA!8u%1Y&|<&c|KMAHc*|KcPUL-GoEg$XDT+ zRKIZUX}s|Jhg0~-WM5YUR*0V*VY|5iB!DhFfjm>T|INdBwwGb>D++ocAO}{ZGj^Db z&nb84(O;zU@vTJ_>_U~Q_6YQnpx7g3#r`>xK7kV6$%;MQ=M)b*#DCG27yZO9Iz!Qx z0r#Xk02ESW;{{O2o!=^H0BOs>R8yUnD%t0J0#J@%@mQuP5qZv_vbD`{w&k(3jbb&x zBmEGH+8M)Tl{~$Z`0jE1teOr9z5e?wcf!&B>|&PKi$Yctgr0+ z0KcZ2Oq(qT7zH6Xl@jOr3VV{6u>Q?JS)^u7P=gr!&T;^6JQ%J9(&t((yEU4Oo*Dvq zBjnPoM+r#kngC}Ii2=R4LA%8+3(?isLi(1>lS*uMZR|JF<)rEcuE|w^Bh(X4lzMRM zB%^JYt5ES+JS8v#lTa-_rirapId&?KrbQ9Tdt2vfVS6Ed-d$wz`08>Rcl^UgW}Fqj z17tnY0o^_lYeCjhTgKcup<-E6)+VmI+;OvG9CPHP+TaO%rWs;d*>}K$-0YvCl^3Ah z3V{Gob_8Ex(C{C3^K+Uee19c2z9OemAzp>#&Io=(8GWBf;R_mz&Bs1ww ziB;8gKukabxofSnPp$;aD2ym;{EZ-Q8ri7l6g?P2+jTvMQcS8qOT_jZn3%H>L9{^y z&Zq6uf~KFVw%np?MKg%EfvR-drh3Cko}IQeb8c$}rF`i*AZ~?{LP@bKb4y|$KyH^Z zKr!bHK+Oe%%D+Rl$6fjo2>}w)w;lT~_l8)-50#wnCfkJi)XU0Tl8x_@fR;M@b}?9W zA+R`T5yhvfIHxw>qMkUJ@+16k#_2nmGLUHTDKiYpdDKkT>6#?(Ckg_#)j}%owumG8;&&nF%30I=|lC3$ejLyMeqga$}$)0J0F{XDxDt(Fd z@_YX+kk-MzBVQiU68%DiW}fX?(7Q$KJOO(AktK)};(xfnvvDYrRn#LO;xz#nLNfj? zYR&NW8DdiuUP}o1>i~81ms?T%X{Yc*`z~%pC-Fjs=hq@eS)^>nkNH6-w0qS_T zz$f*#njrsOnNMyWYh3>m2?&f!eR8u)d+%=x^hH}y4VlUmMzw7k7E~auD7$awLuOy| zV~3LQCIEZ?Mc;@zFGL@Vzto}tiqB>QU?PC%3R;+}Jb{*We~hQp_ZJ#s6`TOzgpALy z4K#e}rj!0LL4JEn_|#rpnNu6LfufxKM~l+CT{d5PVWO1WUK)@c5Ev&M1}@93j@zE^ z0;RU#m!lTFHl6*8*sha0j#277b{}X$Wiiq^{JE?-%WfS4sz;wmo^QEZo-G%iaMaMJu&`bIIN9hYZ5B@53E_xO*3i3QapSefs0URU% zCuoP1*q*K-wlwxC?+H*3=a;~2tt`USwEwyr7~Yj*#=*@h#XzC1I|@3VB!hxBHhQD- zhuWg_k5EnuJIf6cl9>Zxt6>NACH4X1I9$zq+&RfJrCQ1G;qR>o*udDpTxYq4aIfYICT~ zV^$Ctr{E$+7IrFPE=i70K0=g}NA4!piE{J>BaqZ=P@R$p0(oHcP=bYB6w!fQU6v!Q zh}&euyuD)QyzcE$)*8S{eW|iVcVeazi~;P1J3!9WV}}`n#8*n2D3`IoYdo>jAj0R! z&lNFWZIo$xz&E12A=(4D?&S@F%h>r-+iGsH49?F_PV$u0a@LIhi)AvG54A1%0Z>oQ zS_rgVCq0r+cAMDS@(tCSWjk}PZH??oecC3^ZjA~rXqqup-5nf8{0y)WjIRIiMbUUF zzFyX6N9EF``vrJd)EG`Qad}OpbT*6ao+H5_If=)uuRAb#c~!*;^uO~ePJr?K@jE`F$h#n~P%=Ery0Uk~nPU-k-UdNT{EtD^BWjRZ31o+1rJHZ~ z7us<56bBIKV8BQOxa24YFG$hlaexqfFGx|0JB_OYOP=6kPNdVf{OY;M^MDrY4#+d3 zoCrKKjl+Z1AF|`(mWbQ9b#V^oq9f5VfjQn=n$RT&3^~6+El>b`B_SMhz(?7GiKgvw zjvT^U6>x8yu;pS61yWu(I2+_zWb^pML$1(lEy|@h^?;M-$kbA=Uxe6QOgNd0MjywZ(K8q$4>a&u3Yt20H3qsDKzQL>fZ5$7dj5C}fp*OZyJV53U=;TD| zM7DrNrmlH~PM&HGnez*%WZq%$J9k`YC#!%Q`6$kTO$>#jfqfBH8&}ThTiLkuk9O+} zF;#297CN43P@L}etRCPbb5X;x)T^TiU(HeAu${TWx)p1qUC;x0gk2Q&?y@fXd2MxT zMOJK&^8)xg44hyC!p7I-mj-xuM`7hNesC9%QvF959k+FY-~M~CmQ>9EAmsu9gugXXEgc*F$v*HqJ)+R0v{+fKOIuTHm6RtUM&(13i3$ z3?DD%e5iFmJ3IqX{U93V+3?7Tbp2XHe~-13dm>%K%K5nKvjEkC#1t;BgYUgFx0i2g zbW-f~PF`M?Qr0d2nPh<5^GKeQWad$!pjt zC-X8@zpsQ@Ca3*xAnZmopPXDsiRmvS~nK?CJ zd9S)Y{3O`GwTMD+6%_dQkjHr!$SJ@~SubkL_!(Tg2F|R!)AhGITlcZ{+9e4yMSthKB>f+gk7FMfuUWHKD*iJ531DuMk~m+ z{RVjD(X%4GQ{RZ{TjRl_dmDuH-aZidlPg~5TsgbT{5a<~Fdx);g2VrPq0~~Du#bDT zDg4$&FWzh)TF-B~Oi403|Dtnk62BoFQ&1uHptpqH1N-;DlQFHD^WQKi$;N!KX~h+~ z3bpvdHTG}hVgl^YZ6j>yp^=m)T|2&3IeP=4ycc{(&e!31_&cuocJc9sV5S>~C7xzT z5Uc^5U}%MN_rhZ8%a50@^Sp(~RaIJ0Xv3!_nmv+#PlrkCx0451WO~|QcB?l{BcVIc zGv!jHW~y_(^II0>9B39d$JWrp5}4k+D@U(Gt6y#lp3pXphBUF1e$Sz2%eg9$Qu7kIEXWc2PHPaaatE8t)lO8nkeTU&yJrAgDKF8otS z>GAXPu`5p5y0QPf+Da8R=nrB*z^n$JQQg+R z09qiotEnK0)x3|uG=>0WRUwX@&myrjWKQwJ>4QWbyDVB94 zC5Q>qJq0lV-ch>KHP^4m+M>%mTH`{==QSXv=p-jBR^Xzpbt$JG41-DZ1w7JPm;-Y5 zPc@*=@GO&Km_8R-I%G}s;lEFr4J&A zG9OEm6`CFl@du5jy@gWP1dEE?HWx6Qgp&K@sj&%DD~<0<#RD%u?*VFQ1ICr}-%Z17>MDLdEi0QBwLL zWf-Z(KIvLez@bMCu<1@LA%mpLmb8@vL6kTeEy#2dddZ>bj>8(wwS2Q3*Q$xPkIDgZ zXL3a3?grIT*@&45+lVsG)O%1k!Nnd@7{pI064<_OCVV}dxRrWPJ554$K=rs>`H=4d z%F25TxSZY}PzUkbfMG})H1U<*fmihh+Li9gE!uT@WgX@BkVple&@~O=HekgABPLk! zknAP>qLN^3JVCqW<#Z#b+#ngf618h;xA9sk%*;p?X7IG98Z3ViFw95=CQ((DIiZ=j zc33sbBuV{9vgWMVxzSnO`J5-HVlr6JKVluVOiEbJroJwEdgsw+Nr|zqsED12wjs4^ ziZt(MNm?S&?A8e zF?+Nb=Z?8bFODb4T> zOQUh+1p8Hx1^69D(rJg*muIq+HArYe~AP@gTQ&9u!A0K0u*)?ED!Lw?66*XdWF^*@1*7eC~%uB zB_L(H&e;qq2Z5r5t^wTDO(GT~2eysOUlskzT|qb?yy)X&=cze*D2pyHO7)?A4JRK} z`8V#Q2Gk88Gg=^;C{FgbJ`?JR{1pM7;xg4#9n9)+l*$#qc!vr9fW4RWo#p&h@UM7%~60$UB&2$o#Tm~{c z!v)Sawu=Rx>b7qJ*#@48j--xl1{MEh#o z&8_bE5k+HmWkKT8Kfi`(QWc4*<}V1`&Y^?P{CC$!|J`-eO%V2che0Y@%Hr=Dc1P%4 zu=I+uL%DGD+UG0x99eCqzt5|^y1*J5F9wCL3;$iS0kOo)b3YxnFvK*N|BgvPutV5J<`*rEaC;r#g{6{|@20cQ)UVM6FHe>#&#Xt1oRu zwN32#KypUG7FMc_bqpdv4Gv&Ue)X)_i=;&ki?aZhbpoJXb?!O`7(sG+ z;jTG?6y1G%*t(dS{PX0aa4By=7t*D9Ca@&mRz{R!z%zrO7fq1O?U~VC)=`R~>=)h- zR)up40RMwK82=Foij|%?>!`A;*(=~7H-EsRj7FB|?V2Dbt}VkeDZf&2{wGP8g)Q)G z3CMogQ`lkKZn8#Zd`#FhPeOj|86ey|k=_hb(AYP{+^)k6yr1AfPm+mJftaA+ux69n zz#P5?nx5k@vIIz_beG#OO5dz7MR9SVRb&g~I{@1DF;~U7)_-XWB3ow#OW~h@wC>fJ z;amX`pVHF_4gzdFNVJZlM|dp9O2=Cet&ai*x?uhw9%WVzOfPK2Tt^5fXfnTX>BSl^6v!6rUO z3yu^sh}ickT5A_vK)C~tFUfiNO5tJ7_l$-j!kZAI65OE=6COWaLA8_LlIG;x}9v_>G>fbP-@(R2&E^Vz{+0rPVZl{2#XR zi}GJ!k$s=X9viwdD8=imM-LaHBN`eudgU+-ZxRRulT=b3-}>f~Fus^FFc|e}tzXJK z)Qnjugw-W{es291aZ62aZJsA?b*&EeZ4MQX;E{Y@t|o|Yf``KOOrk%jm$Llk57bYa zfW@7FK(qQaw0blyll%FYl#Teg-x<(5JM1UjYRz&=fe0$IhgCKfDhE{Re{d1Le8t4&QtgCEfqh>Y!h ztxxQpjxm_Ti#byX+}B%E+tJay*9ktgs#px)#*H-(-Zba;5A{pp^M)olxEgG{JfXix z`zCpAlzGjY>7}48?Y3H0YZCahIz)3fv?+t%dXw?(IsQH&zboWVySvGP8tA#6VG3O! zB(H>55V%0cp--FL?Rhyti%vm;i?3GU)Gax`D{Zi=Eku#G!Ech#&Mo5OI7@!{&}X!A zh?NkkcBk1_>{4ay926!|jPX6vrNR^2GFMv=&9^)IjC4Aff7{-Q+j7=$G-a4+Tu{vK zES*&roVsi*y&)r~4J%t3VOwog&zh3htsIq+>v$xS(>=xb%*~>t5V0@Av|FT3L47j?EmPer9~=u zq4vcNA&vX&40|W5fasLfyHxanZ_r3id%A@AjXry5WL`_^O!&(YxmWKd6-#f&`Ro67 zIy*8ty~XH!adl7Q-rAam@5f)f#>iAlb=MlFv(;6_sedf_e`1bl#wke$x$7HhsQ#ua zX{+Dlfz~qI#aH;YmTD1e&K#9QTKP_p9qyNnQEdIo>{9K+R*4mH`|o#4C!)_wsY&R9^nDCtp(5E#|_U~^1UMR=*N8z7256J&Gcqo)KL;BC~ zZj;(EU7WxlwLeVK=E>tvjtN{}^ZHWO*B>%481wU5srL(E2{$C4wb#q!&CT8x%f8rV zDtK(O9d;;WXK>KxbI;}-j=|SG;!i*)^D`n`Ke&7?&aj2ZcmB8qMi+<7AGlO*T1YQE z*c5#SNB4?!EP5c^$zGM=IjI1}ZYTWp1#1=8>Rw#vltji24(bivINRH-3g*m4brdmo z3~In{>@#)Ei{0p!Pg@tar~Qb~VA~n0?kjCLI%_Y7F>2vbM+Ig|{!v`a))fwFJKCY@ zGqaL2isPePs?zygi{VBNHP=4OKXp6LwHoB-AXd4JNOL|G*X(ir`d*lVy$gdKb^|=8 zqUKmMN7}@*vOQM^D|D2HhZ(b9c|MOp3sFlX)Gh z$z{iD5>G;Q%IbG<^bnPsx(Z0=kl;P|?g0$ie-xd#vfSj4eplIrClxlVe>2A|@(^9O zK?`0z@F^%m*PoaF=Hcpv)a3r;fYYooj=9QYfEoZJZ zk)4w{l_2aTZT@lbKq|GJj}k2AB9R%{oT`=O6wZ6n4F5TK&W@geK5P73Nb-21d~JW# z_40I88xHDTCU;C5!SS-9%;0It=D2}y@xhFOxqX+G&5zT?5u%m3zk07DudEH;ca6rcq z%e1$S1Z+3U+nzPu%H8>xka^pL(2vQQA}uP@Rc7~nr!pvy{&m4@AW`fs0qH+_CeaO3 z>~7YlFy`jV$jm_z`()m2Oo`wIej+Vei&iH?K|xjjR8IQrgN|RRA0;lPeECY*A|f}R zO4JmRkqPdnXr0?AFS?0?jL4SFnFiD+Iv$48i0EKGfXN|~&dX&1)Um7bRt8%%5CYxQ%X-j=Cnm+WX8tn#XqmH!UGjrXYiH$ zA0b_{!4VfTBUuky?1g@oGml@rd~J5zGx&n@S^2#i3zp!d;AxXNXYEy{_S5h2P>yp& zp%4=O(ate>ZN9hHhjh%);k*0Ih6csBGi4D42U+-E*NASuDr*z;v}G4cnZqTlmP_Hr zfAE=nK@xxd;7_RaAHI>yC?@@OK6d@jh28a!Xb;{D1Ixp=wR*a=mTDA17iK9jm2MttS^{0>z|HQtJLYcoi z{rA*v-Q6iNS^%>gwIt)q4`bu`Ms@Wqen6*{1_daYt_)8VNHZu*!<(S@95h#rn*NM@ zM9WfVY|Ft@c)W%?|1Q?JiY4(h2AOFltrfUbqZ7^bWY^rbM7CPRqi*?a&{e3$CxD`v z&x|`!{NE+#R)3>%i`StIs9vTdS zzLE}ScMy!ZS0_;Jh}M3=rgj%G)ezE{3m!u7nKipji(h|CwY`-bqi7@T}-t;s*Z0X6~}~@Zf~pnPDq& z@a1#C#wrQdbfP&z_RJwz2zPIjL}D9evm_1UDEYhO-C>;M(r3E$gg$LP4ghQuH)~Jn@2r+PnjrC9U3*cVMuLcF1iNW#A^-R<+5?kd%YcdG5X4rmk z+G;g0p%eQ^^s+RTgWQo6v2L_wfYY)6cdV^P%Y zdLxs0H~Zl7aeBiRgxoptGj2cfC9GD`IcYx4{$axM6gHqZtW38YbF}&=m2cM&YcBp> z8^0wou^k02d-N@CBW-lz_AMn>UCj$ujh<@e3i(Zu7{HDC<18g{J&2c?BhqaxqN^^m znT+WW%6=N<4fk3adBL6flb!I4*3I`Afzs+_@}q|vnL|Na?bo~P=e?p-3H>3NQ`3uC zZ93Pw?42v=#+fth6EuoFH=0o3*turP(pnl9`@C9XE`&Q(RiLu#XixE2JPc;)&=68> zzUhUHEOh4;f6Hx=#hFIwf0Z+^X=%Ya%DQZSGEW|yYSU2wiim_q6&c+sik{uOqFnro&Fjnh6&U%;?WeL})-zx74%~|urWx)+dU`Mm<-5uikL^LH;HNi% z$&~cxbv7F2s76tIYD}UuyZw2sPsvp+$4^kZz2JI}@|~@QDhKTc&@s+(a#0a6Y$W*e2c=B0_Lut{*waI)J?mu{`(>T-CV^jCeAf0EVj)WtK(B!v0_uxJ&l>xjTXcJfqsKBh z$Lmz{qNjHVZ*GMut&E;W_JQ0jaJv|w+f{-J;z?j=x;@=VLD@##^oyr$7*sTPl;)qm zzNS)7$|pNPc}1xFKvA_X;3xToc3id?J;$sbJ|Y!@Wk~=hU`P0S`IWGslAOR51nR1K zf@I8{j_@0vV;ojx!cA@|RT#|ZwJWrxHNnN`dH@TehqIe6f3P>waS$BEWjD984rcOg zE;56qkXj+pH~(-W+d8Ezb3KzN>x$^xz?+aO>^z3jl#-@6Vm9iWS2~ei5)SGJl{)ye zhFVi15o23;8R42v<>P*Ca`5t|HT8RIF}7vN0m(aCHt|1d;3Y*F=#WQ<8+|V~Zk?kT zxF>y8RgjiHR-d8`b%=e<;AWYkw^r79Zdn7(aq?>USUK&gMiyVe!(&7WQBv1=Z6UwO z%j%m7C!XpW>Lcmh4z%vAVH%C=^^7-M;$S(a6yk&<%wQ0(zSW?IO6-%I4PPp<%w zR|J2S4W*G}Nek}4)GgqJE(1g?^g;ugzGp*=@aeFIJf5dx z%#Bo^am^p6EkRSiH~aRqp66T3GYKn*gZ05UvNnYJUzHt{ZlWnQgkHe?I0 zh#47yb5%q|8Q(Mcl5zDfV}AKbo|`ND+>a#}^EP$u(Pg;edv)y0IAZ`z430$sQ0SR{@5Xx^W7GJ(~LX0_VWv7<~;3-0uz+Xx$J#3Sy(~fyH zG4M%Y3)jAIaC!d247XKG2c?Tyk6@MSBuV9IVPUGImjjdDpvJ!NE{kH?t3wqo4uC6CjYmW4ACcxB z=oYihelaH!%cyD!p|A8lhJslAZ=Qmj!K`_Q241n1<(bgt<$ERu$9-6~dv>?oqw!Js zRQb7hdt0n?qiDQ6#lZW1gPdxuI-dpn3|3j(vL!`iBCZ&$hE1e3w&o)*vyg~uA&EPq zK2;JUn1|)C%0S*P$rG{vu8;qsP<(LrJ{Lf=>i6=q-G`jybS=32my70lgx>1$>*i2e zmEr&UJo}{WO1-A6PwVr;8M;ek`a;6AsDY$&0pR903EUQJr8o}64O1$pdJe+4t0rBy zO3(w(w@M?JLyrzR0z9RQMSwcr556_)H6U%@&s3UFq&3JOS_nJY!sb;R(HyTOEcgUk zn-C5_tA_S>r9J0Q1&?|Zsb?^D}e}S00Hl3z8Y-NA$4bYSE#c79Yq!?B?tXY{!#_* zyr}9v^ntI#_a+%wVe~ZgHPK*=-mpAdH8jZ87MtJM%P6Rb(2U4AeZrMMh%sVf;_2(_ zL6LiTKs_4V90)Znyt-FFn&LbOcmhHUhDqmDt!m>22bKYmi>h{*_HG`*OpuAVd5qv) zSPPweZAQ@6;vSLJgJY&`B%|^ves{VnM*POwz#N~k_m>}L2Y~4ScoVQYQ-wV#9F7(T z*UOjQUugRU=$8-8wrq+osRsX zKLDD&=mSLeg|GlQ>QSK#Z^_ZPGfiGmN35MX>BTR$Os)NbhreN*`7g-l5QG`#L&i$n z81uz>I5|9AxB{Bq_ml4|yq|V%}Zn^RPkNKUSxW zQ{E%$QMat)Z`(7KUN}4K^k^TkfHG|`p}2&ssXA zW?U6>j!rB#h#M+ez6jTTR?&&=#j7o#x@HE_gw`2%{Mzy_G2OjR8xbT{ldS_0WtOFn zOAaEJ1_LU@9mVD6Lf#!!w5TsR^qisTs@!|)8Ywv~LT}^K)(hzqfk9+YX;~_&%{!y% zWl|@m;oM=p3?>KaB(7(N>PuUEcCEGL%wKo@*Afd{cb49)@+YacymYaIk1iSY@wh-N zkXw4cirw;uvK=j&9GfCWHZ#P68^lo3508X@DIYN=e>OOc~U}mUwE3} z{d@j~6~Nc7+UEb};xxe2ZzVUQ$ycNSS9&k;3lf z>f-?aGKxc7Yq)t%{H<0LZ@!vz`(1tU?~D!}jnz}5zUOb|Y*Ud5BXF3Ysx6s%S~{2m z^f7|cWBp8)L}gVE>gXFY{z;}%JNm)b-E6A=T6rTAkI}FH zvCo@ag{cisr9OJG{t)UU6}KLs$yl~q(ATEa{cFGO=J1$qC0#n`6<3$fq@~U)o_GE! zGqEyydib(*{#i?-UXuLk!sw?dQZGxgknR0@Ld}oL$Jp*F0}8!ZJc?Etz{m3)62;DD zyJ7rsuX8Yu>y|-tcI%l~B|7T$nrO0T(5Q%4X%mwa92`Q>i%k1V_E?InS;8P@G^{}~ z<}(R2f1u{$f+%B!OS}f1+5MN($B*IYV(vH)ui`YQqdBZ$Czbrm5vdhFSsjb!zpmOG zHPZ3IC#`eO3F^8jyVjJQTWu41tHlbTm!Yne^z1^3721yQuj(m#A$*+V<|$U#MNoQ9 z9ibOx`Wrsf71RAXk8d-K5n4ZoqEogC;MqZcr0snFAIT6e8dTZ;O>Tk`|ZMa27#cV%LRA=$E z4i573p0u)U?sWMa*VfhkGFekvxL0E33Nf?WCVmz7KHGTXo%F8S8~yDrUX2I6OonL- zYLr&_Sj^B!zD$(Xqp>r94@&PNuV!aGM!qoDv~>O7^O<7@&KfIu*j)fODRn!&N*P7W zuq4gaRF`VI`uyO#(Kxy=x@8S%Qx(^6pL7YwI;QKCx(;KMRxs(!ho+lzRJo*gQ%cze z_I?m@hcj~sAdHzZ1V(k0Roj>%1oZ}?*}|qCJ2jQCTfEW?9=qZ@`Q&Ja_kTv#e0#)j zDu-tP%wXW!5z}NzbvB*^#CV+RE}mdberW!sua+G?(Q`9so0c!zzA3M@bTgHENlr zeMt=?Ip>f66Y`=_N5W}HP-Ftn(O?BoU99lM%&eEZzI`&GvBJH8{0c>=yy_)tGFpef z9X2{tR;`V`Z&{D=%IT2Z#YawZ-l|Da`u{{q0pUsM*sKn&ZfUew$^c>lN#(b{x@}U4 zED4uN$1M-AHA9>WTQlf6EoyeA7Vlfs+*tpf(2UiL*K3SU*tOn3Yd^?0uu%5xYsPkZ z+i2#J5ESGF+f-T{$5Y`rtAk!XGj8}fWbq2o1E;sP8^^MMj~ozJEuzUOL3M&PThYah zaiETlqsW83$>Hk`j{exI2=L25^|!L? z{;467un4mVG8gj}n!pb8L8lqAh9`_CRXN9W=;IqVr-t7oGPi-q;a%;?or)Y3}2{wV^6)H;l5G;x?EF_7t>E)iv-_r7}g@O{?}jgH;6RLM3b{2_H_$ep+RP2E@a7}rWTXFD2gRh@py8@pcFs^)bvxZ`37nT{cZ9R0?NeH>Bqdlu zixC=MI=@X`evNFNe{U9Za=%sPrYKs4}6r~WC^XyRu`oSZoonKYus7M5vxhq)`Sbu?r; z9MGAy8frWF>CR8UKdz(~qM2l5)WrYtGI$aPvbZmjmY**7e#I|UXg0T(y>2x<8{UKB z4Zx)(+RU@&zyl5a#l|;nXV@u)MYVet=6QSt zJXfQ_`+ZVFOAcyw&X=``7P&ge_cG=m@t6tLVc^TD_pD&@eF;LJ#2A&s!oghhfloBV6mR(~sEr zF>ViE)wR!kFb?vCci*y%x$Kq}e$H!kxlQQy>JAd>+mrqEOGl?!(rgEsmtx?C6~pw>3*q7$GjC~P21`S?$(LM;mNv(Ja7x{xcK*i91~`+xB=!MD zDXd{oJ4gL_%gbuB(o;b#E;(Z#(OLlU$(eIs>#EVw$>APlCaN=mN1xHg32R^L0+TkX z4Cn8UwXWv7`Uc*gHbyVYtp-V0iNOg^#yh2^8RTnW4&R+;V;^&PZ~#Er(8Q}_FX=$+ zFG1#iKez?s1tkp+G!wQ;kMA?gErKDNEoC1p<_=?i?Z$x5l#E>V8;ZWxpf_s;oU0=z zjC!+Wj~Ejd-;J$(-1GcCRF26*UDxgzX z3wtmO>8sdfY#klP3RrHU_W_ietp(3jR%b9rPI%pEWG!Iq#xUOf6KqsBGJ!M z!_OgwZYxK%Kh;c%as3?QN^STqD&xTa(=ama2FF^=OZJhg?gl zButrvq8`v1d^9$Dx7i)HsF%!Owykjl5Bg~}PG(75*wCPHkg?}L#M&t{ZCXomf P4=H066Py(Xk01L#$465H literal 0 HcmV?d00001 diff --git a/caravel/migrations/versions/960c69cb1f5b_.py b/caravel/migrations/versions/960c69cb1f5b_.py new file mode 100644 index 0000000000..d304d539d6 --- /dev/null +++ b/caravel/migrations/versions/960c69cb1f5b_.py @@ -0,0 +1,24 @@ +"""add dttm_format related fields in table_columns + +Revision ID: 960c69cb1f5b +Revises: d8bc074f7aad +Create Date: 2016-06-16 14:15:19.573183 + +""" + +# revision identifiers, used by Alembic. +revision = '960c69cb1f5b' +down_revision = 'd8bc074f7aad' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('table_columns', sa.Column('python_date_format', sa.String(length=255), nullable=True)) + op.add_column('table_columns', sa.Column('database_expression', sa.String(length=255), nullable=True)) + + +def downgrade(): + op.drop_column('table_columns', 'python_date_format') + op.drop_column('table_columns', 'database_expression') diff --git a/caravel/models.py b/caravel/models.py index a899e82d14..46006e7459 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -445,24 +445,6 @@ class Database(Model, AuditMixinNullable): if self.sqlalchemy_uri.startswith(db_type): return grains - def dttm_converter(self, dttm): - """Returns a string that the database flavor understands as a date""" - default = "'{}'".format(dttm.strftime('%Y-%m-%d %H:%M:%S.%f')) - iso = dttm.isoformat() - d = { - 'mssql': "CONVERT(DATETIME, '{}', 126)".format(iso), #untested - 'mysql': default, - 'oracle': - """TO_TIMESTAMP('{}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""".format( - dttm.isoformat()), - 'presto': default, - 'sqlite': default, - } - for k, v in d.items(): - if self.sqlalchemy_uri.startswith(k): - return v - return default - def grains_dict(self): return {grain.name: grain for grain in self.grains()} @@ -525,6 +507,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): offset = Column(Integer, default=0) cache_timeout = Column(Integer) schema = Column(String(255)) + table_columns = relationship("TableColumn", back_populates="table") baselink = "tablemodelview" @@ -607,6 +590,12 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): def sql_link(self): return 'SQL'.format(self.sql_url) + def get_col(self, col_name): + columns = self.table_columns + for col in columns: + if col_name == col.column_name: + return col + def query( # sqla self, groupby, metrics, granularity, @@ -661,7 +650,8 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): metrics_exprs = [] if granularity: - dttm_expr = cols[granularity].sqla_col.label('timestamp') + dttm_col = cols[granularity] + dttm_expr = dttm_col.sqla_col.label('timestamp') timestamp = dttm_expr # Transforming time grain into an expression based on configuration @@ -677,18 +667,20 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): select_exprs += [timestamp_grain] groupby_exprs += [timestamp_grain] - tf = '%Y-%m-%d %H:%M:%S.%f' + outer_from = text(dttm_col.dttm_sql_literal(from_dttm)) + outer_to = text(dttm_col.dttm_sql_literal(to_dttm)) + time_filter = [ - timestamp >= text(self.database.dttm_converter(from_dttm)), - timestamp <= text(self.database.dttm_converter(to_dttm)), + timestamp >= outer_from, + timestamp <= outer_to, ] inner_time_filter = copy(time_filter) if inner_from_dttm: inner_time_filter[0] = timestamp >= text( - self.database.dttm_converter(inner_from_dttm)) + dttm_col.dttm_sql_literal(inner_from_dttm)) if inner_to_dttm: inner_time_filter[1] = timestamp <= text( - self.database.dttm_converter(inner_to_dttm)) + dttm_col.dttm_sql_literal(inner_to_dttm)) else: inner_time_filter = [] @@ -909,6 +901,8 @@ class TableColumn(Model, AuditMixinNullable): filterable = Column(Boolean, default=False) expression = Column(Text, default='') description = Column(Text, default='') + python_date_format = Column(String(255)) + database_expression = Column(String(255)) num_types = ('DOUBLE', 'FLOAT', 'INT', 'BIGINT', 'LONG') date_types = ('DATE', 'TIME') @@ -938,6 +932,39 @@ class TableColumn(Model, AuditMixinNullable): col = literal_column(self.expression).label(name) return col + def dttm_sql_literal(self, dttm): + """Convert datetime object to string + + If datebase_expression is empty, the internal dttm + will be parsed as the string with the pattern that + user input (python_date_format) + If database_expression is not empty, the internal dttm + will be parsed as the sql sentence for datebase to convert + """ + tf = self.python_date_format or '%Y-%m-%d %H:%M:%S.%f' + if self.database_expression: + return self.database_expression.format(dttm.strftime('%Y-%m-%d %H:%M:%S')) + elif tf == 'epoch_s': + return str((dttm - datetime(1970, 1, 1)).total_seconds()) + elif tf == 'epoch_ms': + return str((dttm - datetime(1970, 1, 1)).total_seconds()*1000.0) + else: + default = "'{}'".format(dttm.strftime(tf)) + iso = dttm.isoformat() + d = { + 'mssql': "CONVERT(DATETIME, '{}', 126)".format(iso), # untested + 'mysql': default, + 'oracle': + """TO_TIMESTAMP('{}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""".format( + dttm.isoformat()), + 'presto': default, + 'sqlite': default, + } + for k, v in d.items(): + if self.table.database.sqlalchemy_uri.startswith(k): + return v + return default + class DruidCluster(Model, AuditMixinNullable): diff --git a/caravel/views.py b/caravel/views.py index 419f8eadc4..57cb39819f 100644 --- a/caravel/views.py +++ b/caravel/views.py @@ -187,7 +187,7 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa edit_columns = [ 'column_name', 'verbose_name', 'description', 'groupby', 'filterable', 'table', 'count_distinct', 'sum', 'min', 'max', 'expression', - 'is_dttm', ] + 'is_dttm', 'python_date_format', 'database_expression'] add_columns = edit_columns list_columns = [ 'column_name', 'type', 'groupby', 'filterable', 'count_distinct', @@ -201,6 +201,24 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa 'expression': utils.markdown( "a valid SQL expression as supported by the underlying backend. " "Example: `substr(name, 1, 1)`", True), + 'python_date_format': utils.markdown(Markup( + "The pattern of timestamp format, use " + "" + "python datetime string pattern " + "expression. If time is stored in epoch " + "format, put `epoch_s` or `epoch_ms`. Leave `Database Expression` " + "below empty if timestamp is stored in " + "String or Integer(epoch) type"), True), + 'database_expression': utils.markdown( + "The database expression to cast internal datetime " + "constants to database date/timestamp type according to the DBAPI. " + "The expression should follow the pattern of " + "%Y-%m-%d %H:%M:%S, based on different DBAPI. " + "The string should be a python string formatter \n" + "`Ex: TO_DATE('{}', 'YYYY-MM-DD HH24:MI:SS')` for Oracle" + "Caravel uses default expression based on DB URI if this " + "field is blank.", True), } label_columns = { 'column_name': _("Column"), @@ -215,6 +233,8 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa 'max': _("Max"), 'expression': _("Expression"), 'is_dttm': _("Is temporal"), + 'python_date_format': _("Datetime Format"), + 'database_expression': _("Database Expression") } appbuilder.add_view_no_menu(TableColumnInlineView) @@ -388,7 +408,8 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'table_name', 'database', 'schema', 'default_endpoint', 'offset', 'cache_timeout'] edit_columns = [ - 'table_name', 'is_featured', 'database', 'schema', 'description', 'owner', + 'table_name', 'is_featured', 'database', 'schema', + 'description', 'owner', 'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout'] related_views = [TableColumnInlineView, SqlMetricInlineView] base_order = ('changed_on', 'desc') diff --git a/caravel/viz.py b/caravel/viz.py index 007cc53db4..55ead440fd 100644 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -146,15 +146,34 @@ class BaseViz(object): self.error_msg = "" self.results = None + timestamp_format = None + if self.datasource.type == 'table': + dttm_col = self.datasource.get_col(query_obj['granularity']) + if dttm_col: + timestamp_format = dttm_col.python_date_format + # The datasource here can be different backend but the interface is common self.results = self.datasource.query(**query_obj) self.query = self.results.query df = self.results.df + # Transform the timestamp we received from database to pandas supported + # datetime format. If no python_date_format is specified, the pattern will + # be considered as the default ISO date format + # If the datetime format is unix, the parse will use the corresponding + # parsing logic. if df is None or df.empty: raise Exception("No data, review your incantations!") else: if 'timestamp' in df.columns: - df.timestamp = pd.to_datetime(df.timestamp, utc=False) + if timestamp_format == "epoch_s": + df.timestamp = pd.to_datetime( + df.timestamp, utc=False, unit="s") + elif timestamp_format == "epoch_ms": + df.timestamp = pd.to_datetime( + df.timestamp, utc=False, unit="ms") + else: + df.timestamp = pd.to_datetime( + df.timestamp, utc=False, format=timestamp_format) if self.datasource.offset: df.timestamp += timedelta(hours=self.datasource.offset) df.replace([np.inf, -np.inf], np.nan)