From d2dbe2deb1f8efb84c8c37c582381705fe486648 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 24 Mar 2023 21:43:28 -0400 Subject: [PATCH] current state --- database/build_maps.xlsm | Bin 0 -> 16638 bytes database/deploy/reload/dcard/curl_dcard.cmd | 4 + database/deploy/reload/dcard/d.csv | 139 + database/deploy/reload/dcard/extract.sql | 24 + database/deploy/reload/dcard/map.json | 25 + database/deploy/reload/dcard/srce.json | 76 + database/deploy/reload/dcard/vals.json | 4299 +++++++++++++++++ database/deploy/reload/hunt/curl_hunt.cmd | 3 + database/deploy/reload/hunt/extract.sql | 27 + database/deploy/reload/hunt/map.json | 25 + database/deploy/reload/hunt/srce.json | 42 + database/deploy/reload/paycom/extract.sql | 58 + database/deploy/reload/paycom/load.cmd | 4 + database/deploy/reload/paycom/map.json | 64 + database/deploy/reload/paycom/srce.json | 392 ++ database/deploy/reload/paycom/vals.json | 139 + database/deploy/reload/pncc/extract.sql | 32 + database/deploy/reload/pncc/load.cmd | 4 + database/deploy/reload/pncc/map.json | 455 ++ database/deploy/reload/pncc/srce.json | 67 + database/deploy/reload/pncc/vals.json | 952 ++++ database/deploy/reload/pncl/extract.sql | 30 + database/deploy/reload/pncl/load.cmd | 2 + database/deploy/reload/pncl/srce.json | 73 + database/deploy/reload/pnco/extract.sql | 32 + database/deploy/reload/pnco/load.cmd | 2 + database/deploy/reload/pnco/srce.json | 60 + database/deploy/reload/wmpd/curl_hunt.cmd | 3 + database/deploy/reload/wmpd/extract.sql | 63 + database/deploy/reload/wmpd/srce.json | 148 + database/deploy/setup.sql | 2150 +++++++++ database/interface/import/map_trigger.sql | 257 + database/interface/import/srce_import.sql | 215 + database/interface/import/srce_import_do.sql | 184 + .../interface/map_def/srce_map_def_set.sql | 110 + .../map_def/srce_map_def_set_single.sql | 98 + database/interface/map_def/test_regex.sql | 222 + .../interface/map_def/test_regex_recs.sql | 211 + database/interface/map_values/map_rv_set.sql | 64 + .../interface/map_values/report_unmapped.sql | 249 + .../map_values/report_unmapped_recs.sql | 257 + .../map_values/srce_map_overwrite.sql | 261 + .../source_maint/srce_build_view.sql | 33 + .../interface/source_maint/srce_delete.sql | 52 + .../source_maint/srce_overwrite_all.sql | 108 + database/interface/source_maint/srce_set.sql | 144 + database/readme.md | 128 + database/reports/all_map_return_values.sql | 245 + database/reports/colateral_balalance.sql | 19 + database/reports/dcard_bal.sql | 8 + database/reports/key_list.sql | 34 + database/reports/list_maps.sql | 19 + database/reports/loan_balance.sql | 12 + database/templates/regex.json | 25 + database/templates/srce.json | 82 + database/templates/strip_commas.jsonc | 24 + database/templates/transaction_type.jsonc | 45 + .../001.extract_schemas.sql | 30 + .../002.schema_build_sql | 33 + .../003.srce_delete.sql | 52 + .../004.source_overwrite.sql | 108 + 61 files changed, 12723 insertions(+) create mode 100644 database/build_maps.xlsm create mode 100644 database/deploy/reload/dcard/curl_dcard.cmd create mode 100644 database/deploy/reload/dcard/d.csv create mode 100644 database/deploy/reload/dcard/extract.sql create mode 100644 database/deploy/reload/dcard/map.json create mode 100644 database/deploy/reload/dcard/srce.json create mode 100644 database/deploy/reload/dcard/vals.json create mode 100644 database/deploy/reload/hunt/curl_hunt.cmd create mode 100644 database/deploy/reload/hunt/extract.sql create mode 100644 database/deploy/reload/hunt/map.json create mode 100644 database/deploy/reload/hunt/srce.json create mode 100644 database/deploy/reload/paycom/extract.sql create mode 100644 database/deploy/reload/paycom/load.cmd create mode 100644 database/deploy/reload/paycom/map.json create mode 100644 database/deploy/reload/paycom/srce.json create mode 100644 database/deploy/reload/paycom/vals.json create mode 100644 database/deploy/reload/pncc/extract.sql create mode 100644 database/deploy/reload/pncc/load.cmd create mode 100644 database/deploy/reload/pncc/map.json create mode 100644 database/deploy/reload/pncc/srce.json create mode 100644 database/deploy/reload/pncc/vals.json create mode 100644 database/deploy/reload/pncl/extract.sql create mode 100644 database/deploy/reload/pncl/load.cmd create mode 100644 database/deploy/reload/pncl/srce.json create mode 100644 database/deploy/reload/pnco/extract.sql create mode 100644 database/deploy/reload/pnco/load.cmd create mode 100644 database/deploy/reload/pnco/srce.json create mode 100644 database/deploy/reload/wmpd/curl_hunt.cmd create mode 100644 database/deploy/reload/wmpd/extract.sql create mode 100644 database/deploy/reload/wmpd/srce.json create mode 100644 database/deploy/setup.sql create mode 100644 database/interface/import/map_trigger.sql create mode 100644 database/interface/import/srce_import.sql create mode 100644 database/interface/import/srce_import_do.sql create mode 100644 database/interface/map_def/srce_map_def_set.sql create mode 100644 database/interface/map_def/srce_map_def_set_single.sql create mode 100644 database/interface/map_def/test_regex.sql create mode 100644 database/interface/map_def/test_regex_recs.sql create mode 100644 database/interface/map_values/map_rv_set.sql create mode 100644 database/interface/map_values/report_unmapped.sql create mode 100644 database/interface/map_values/report_unmapped_recs.sql create mode 100644 database/interface/map_values/srce_map_overwrite.sql create mode 100644 database/interface/source_maint/srce_build_view.sql create mode 100644 database/interface/source_maint/srce_delete.sql create mode 100644 database/interface/source_maint/srce_overwrite_all.sql create mode 100644 database/interface/source_maint/srce_set.sql create mode 100644 database/readme.md create mode 100644 database/reports/all_map_return_values.sql create mode 100644 database/reports/colateral_balalance.sql create mode 100644 database/reports/dcard_bal.sql create mode 100644 database/reports/key_list.sql create mode 100644 database/reports/list_maps.sql create mode 100644 database/reports/loan_balance.sql create mode 100644 database/templates/regex.json create mode 100644 database/templates/srce.json create mode 100644 database/templates/strip_commas.jsonc create mode 100644 database/templates/transaction_type.jsonc create mode 100644 database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql create mode 100644 database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql create mode 100644 database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql create mode 100644 database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql diff --git a/database/build_maps.xlsm b/database/build_maps.xlsm new file mode 100644 index 0000000000000000000000000000000000000000..2b5222846fc361297c39877c8b6a1a30dbaa3d91 GIT binary patch literal 16638 zcmeHu19v9fwr=cnY}>YNTOHfBZQDl2wr!_lC!M5YJGuGxzUS_}`+WBooKx=@wMLDq zIji1P&%~NFA9*QY5M%%_00;m800Mv?a$LnxfB*nupa1|!01!Z0Lbf(e#x_p6%IilhAVB1K06<^Y|L^uccm^t!r(_4{VY-Q~zo}H0jqUisYmWHm65#{F?T~C? zrjRON4?jG#Zl(cPQ-x`-m4x@;jXYdg%(WWCgj-!DvnK;9rs;&{u#OF;apu@4l;OL88Rujw1KrMk3ou5p)fsimDW%7 zn|e{THYmnpwp)sJ*?~mNB}#c+ob1dTT&SZ#>w@#b|2C_#Zx2DSJ2XV*N3hZ*!PGFp zct++8NkizVP?? z2@D|rA4c4yLQiz}bw>KjMxnoqSl7YW%8`!tul@fT@Bd(n{kOkfnII$gWxOGm;%}j& zk4u|z@PbmV0^%J6%07M)8*ok0g~V8!J!IJM%9w$`V!mBIZ{zEm+|j>B2_E*CD7p{ft;Ad#n zFd>nOu|rUK()~4uWi+-7o~i+t1eMOKLTX#s@_r}IWcn_rl$^lwg>guq&8MS`IvSX+ zR(p+F5!^pxswkOpnAaNQIB*iW>l#`0-U?^-AiaCh$YzZy5;7w`G0upMkmTL_YBn-o zPh`6fvOx}1j9!feMBGsiEfW#^Qs zt!urN_lx(tgCa`|2FWP=E>Ek3bmdE@?6w6ljE2tI9nB~bB4l|cw z2^SH`^Xk@OV%yBTH$JwMV#KScJ*mKJB*e+*=5KKL94#nMNjbmC-HNOjnN=qZiQ^KZjCd%nTBM0x1OQ>9n3hSAm78k z{4m;ALIwAQpD&pGJHRTH$86T=;k(d|`QSULG_A(!^*FA6{^VBx`H}b?283BeBZxwg zx3(#)Y+i3N!U8j@+R?X!qqJd^aEBj z{7P#&*oLNLcOz9RSPH5>Tqc_`pnRo*hI7!=gC_Zp02#qd{Z{(u>b;APBcqZ;6Qv6d zt_0CK3$vm zi6uT=Ey}6Gnz3Y#0YA)JsB@8kD}J(ky9YA#Ub4YsdH-AIDA2pq&HIH~>o3$I{}Z*2 zX8I1sMoLZ&<~F8|fBD8JWd+MG==$KD@nK`0<=#x)^?XTTJM!q<5hCcfa0kx{jQbw8$9)#!6G_gkxZSvjUMeTFQy^3Q??Q&&8L| zh~S~2>@zt|r*Pu>e9|vZ0=rc$!{D^g-d0Z;kEtxOkYAEc8_iE3S^7okD7BO@*TuS;&k!JbbqPCQ+dIUPA%~JHcfT=Wb0$Z_ldDPBi`BX_f8ct? z`hkRljkO7IqXro>PXov&n+o5~;17Lwn(MSTJmJSLSb!t!$}^bV;f|s;jPPv_JZ^~>^@22D(_%lo7~NV4nT z)dTp9P-{`ziYSR`WiI}*NsZ9%LJ@btp()9qsCHP|@AP=J(T21Q0e$V!o>RMeKGbJ! zb!?%dTqzDp27Mh7BjrYU8SS*Pi`S+>YbUL<(h)Yniz^|ixm!Or4JAFN!lyHh7sM^@ zWSh6GD0`JJ*870bl#NQJWdoreMxh#d05UDyf5xO*%`j;;>Z`CYo~o0ZZ@SB!W;maT zm-O&_R9U1ub9}=7otNxGgYT9F2LKqx|7+R#4=?FtW^8Rt_m4fpUvN9qn2N+=L+(L) z;e&B>ePh{=CE41Uu}a(|Hp))IZfZVLP-S9F=)^(-BH_4KBrhsX6}06^n&$_G?YfJF zByK#xnT1g{b zNb$8~CujDO6HkglFov{whNga#TF>=yK(HQz7+C>+7PiVy!IR)vr$ils2#Ig)HDUMP zCXE^q?>ja=Xi6{XEBct5vb`|tP5QBedPZ1U9+coz7)`mIUu>Dd4=;d*(OdOR{muvzeDX( zWde0;R#&^B4W~RdcsWnmw>AV1rhP{Qx<-J4_xnI-wn?e!>2e;juGYY^)VYp)%blgX zu!KWv2yJkag=^ayic$Y@4AK@%pg>mw#f(B6Y{i+PlC(M@+o5LE1tTwjACMxk#NOBt zjNB|leQP8+$Ijq7$pQqTZ3rT%UaK+%tfb0Q4X^)42~k8dJ!4U_wqCe8mg0k(!Qhy5 z^BYy(5*t2U4v*)@)yZvW50CHN@$-OgeOnWylnpqElG4RetnTOgJKC0R&&Sp8Yqpa` zvg`iAp10?T)Sl04&imTE4HUX=@0**GM6_-1`y;tnoV9gqoU|)~@Vcw&UBgS#NPYir zuh)Ql*@N_gg9-cy$1Y_T6Qe*nTKK>Foomng*Rj=yK|8s7ELYp{uG@%X(rZNPKsOAd zT+m%Pcgiy+uCDBIq&E|9+rm4-XX7m|XeBEF9*>yv)bDCVFnhQ?nt8pMhD+MG16)}VquR6irW`M>L96KoT2xa0zFVL z``=!1wFg-fq#Tat+al4K;(1(FqUE_A*Q66h2YpDt5s1XQ21=lL#jbJ*^9W(f*E4Li zKsNWkn-I2w9#~`~k%he&Pvg{eMx}S*p*aQAMjEeiH?K5Jg)?z+-u_5l>p)j>iW)}u zY~giOpVeoYIBJ_yBh*XAIg_lU8l6_T&(Z@lPAIqhN#T$RL1p}Son&2WYHJ_hhDsie zuwC?wDZDAE&3>4r?2#SwO=gD$=n(H9a)DwQ^q4-OqetR28bZS~)dTgyOWTgh$F^Za zPj`#wyc-0=%K`)cq2*x;UcoYeTR#K>3I=U6APnW>yKr`erqxAK=1L|j95Aw|Y8O)C zLG;i}cs<#h>o~3}pd%P)tk(5w5@#m4@sY591LPwfwWWty&PK0YLZ#zRcNIsSrKiu= za+3ue-Wua+%k*2#;roYs0gLobFaL4U%bVjIxz4&nCW54k3DBVqL04~(&bcUX`m%L} zcCt1Lz-hv#43;$NjU*zPeS1G<84qz*8gcj>5i*4RzX&kC8Y1_UbF1br~?k;*q0{79= z0E5^HZNnW98rP2+hQ)-D#oCQUr#AYHPBvu8g!gLSI3E@3@3Yv-`Jueyj}zpX9V$e*%7 zH953sL0FI&`xI)FfhS9Hm9=2K=e^@o8R{KUcpG&CJX&-#fBtQ{mm79!o${6Lfq(%3 ze8uO#7AcNS?pDVC$n%PnwZ8H^FYVyZbv1LB}xCBw7TyVFk)x~qSDFjdlsp?HE<%ey`!Avl$4DI z7`%*;H8+o!il;FFo0k}l5co+`S6gi+B4IR-(u$t>SS34hRnbM?lpBbe8KP8w>LvsX z>a&w2I0SnJB~yQb3oG8bnFT0Lgm88r26PyP2FFvYwQv=~ZbId)i85&xux4bsMa`z` zfW&t)ul-)r1|Xkvo|1Y$+L|;FnYOBk3oWlcH%9_!a@%=7K6ha>`TvV@GSCOG}0XZDb4EQ)oH2PtKt% zg$;4Wke&Mbo?w>xP!0h44Kmb4ngS@fSsY#rY>*l~7LWANBivt}NTtro~a7vAd>y`$7a%t|XiTB7SxPr2Xa5z1BNSM^!xM zs9^g#4AiE3M=$=YSs&3CWZY*T)29YqdcZ^`{=9R4 z=#eEn2w;>wGhjZdM;xLdg+yH0*oCxwyZ;pQ3I#CiB;YM z(=K$rE zna@_AD{XM)#b!%0IasIu2hDG z0rk*S^xDM=$nH}BHgDzD4|7W&k6R2OlQ8Ho&Wm&_za}?0T&J^_o9KLe9dPda21Xyc zqy(r{K!SKCa@V%r6w}aOUQ_%ER_;nl_C22z-G<|@UYBP0w7Fk)Tg?3Jqk03-w4ox| zza2Pm)T6Gj6ei&5!NsNHwVS}i?sj5>60nv^kStS3B}*2bWi*pjQjhB6VtrVdXsO>; z6-Rct850l2U;uxb>4VR$Wmq0E9A)gb7+-DOYywDctjFN+k%@vHXgc^_Jm7w z6Au#X=0ZVUuz_?6B2M|PzdsVzE>`ExH~g*HfEyGR6#2D}C1;4zB}*zF*cZO;!uu2J z$7(1e%Dj(aV57cf-MgP!u}U3az-KnI)J6W(O^L#bj|{&Y2pC_9n+a{U+X0_X?}HZt zaALdPoAKESZuc2T0udE~uel(G_$olLayE-P7} zrY1(U`5|XiT`*7fUUDS1XURyIXI}3xjve<2LTkzJplS}SKfWV2Q<& zR^{Q@!Hr}!F()$Ew7a4hOFM!iT-7BH)Ny;QIvOz7V*PucPqS?g{PyGh$ZIUM93{6pJYaim%uaY$OWZPs*LblAxT*Pwa zD-_*(w6@->O-`UAJ$$_L?Mk|thfk#pH!LcOR|LFI>vgt&sx=8s^c@p5B=sf;Hv4K( zLCOg2p|WKIw&9;K5JNAioyRVo(!PXI-e;_+!|rUpT)47G>DJdxchLm&r@zdj*U&}V zQ#Ga-fZtn>dJxSl_FQR46&S;_cd$akNcp^!3WP@v;0`p%hpRa{O5kkexsE+mwG4GK zF&|l{jMwf?)i5GYE2j^;C7VZO0yH6Wl{qN=%v`6WN{PI+1~dPL*iy5Ear1^eMNh4) zE7EmR&{rEY`0E4IEjCl5KS5ahVi!7?DY%$HM;&uNmYKpuekG-{#RbBWSj#vNb@{;n z+oLvjpY4SiHhlY-3BQI$s9xP#34FFg>u^B^&E?bxFh#Rtq^5p(lRA2R#0(~wiKT== z#}ISBnN>3aXR(#$^c)A1&hQgYd|?hV^9m39THrTLvN<8*h@vS!EwU%flz~bP`T$>O zAni->d{U8jWUWK*Ds{Aocn@}bj^BJbC6W?WTxvRK8>FIEA$T$>RXwe0bJGPsP-TNx zN%2+>E|V9rkK9zjTf}5Njq;VOtCbUJuAYRA%5Uc%b5_fYA$R(*OeB1?L?WhsPeIY6 zMWIho$iNPO`BfFTDc?cHhHw(-Oqs+73c1$P`(xQ+&F+ODARI9==u2&*okE%hi$U^e zIH!i)Hw?|uD&B>clTQiSPjxM?n2w`&pK6&d=f^HROyIGT8<8H3&(ZBHH<-fddF@-O zJ=h`eoyklNFLr`q6{qYZPx%^n78>v=o0m#4RG!0g>`TV4&+EJ*d2A{6;c!IW!#D@} zjG{!ft2`UXPF;GBq*r6dTjM(wADnvVj)oVJpDWifd8oM#HV1LS*_H@w<#%|n6?gfq zwHj||bJkFanV+?>g?_ZN{7$DP@SQY)M62M!u3tL+$g3&P0^Zz0HzYRBYPe<3rg&p` zM!xZ&v1ddPf1)N{Kdh0Po(#O*<5!7d;|V9mT}Zl3bsm4dHD4{lOK{|kmITC@_~a^d z(R>{;_WF<-K}(8Zj}~Mg`-QYldnfEl7z#9lRmwo_0_;T?ir0X^Wv`6dNyi>9GFP#P zb`iYuyCGsJ^0qh2&<&QWr3-@N>JaRi7^vQ8_cvF>0dDt+_y@^0?-PU9Wv!29FB=Q` zR*{-yZQhI*o+4)j_|3`iFRJVc-@lF9;m;k*uc%)En;!rG;h%Ba#Xw))!Pdgq(23T- z+~!Qn(@lBAIoH#bqzV-hRYMQH|2r`}qWI0`QXmRG^|=rbQCviAZ?ly-N!4V5NX1!o zEj?IG(3_;45WS2QSu{>)bIdurUWX5JIE|qqSM)kNtgdB;O`v54=X>F7N~5fm%PW`T zkz~aZ6!EG=x7RfapUFjf8N8@$ekUAmm#e zJt>i3*suXZcGR)y%w;?XZNM_K(-w*!f;eE=j7%>OJ`O!_gC7h&QX1?eAa(C|eCSay zGwewrcDzUHA4&}J#PUNc{coyVRhT2DtO7(Q_X|%l56oS+&N?+ZzFs-jR%Ewz@ax|w zKs4Y^cW#c;8O1oHQ_6fYgpg(L2OeAD(Je*ab>4#0okbPXqF>Cbe>0{@5}+D+Xf4XU zWI3^S4bS*mz$JVBc$RZi&;2ao69*rh@twyj0^U2~I{}^tlmdA1q3Gwv$5RKR28Z%P z;U|y>sRU649O8$Rhx!g%1|Ts7mIQdn4=N8?33kShQVXUIT?t(7f8Wc(56f>j1IP}D zwgdLt&qEKm7NHe&1%w$O)1QW)XcpuRp%%pqtc+)wzzVP#n9J|fAIlHRUv-Cc$0QHd z44j)Ed=}CPP+63%UrTr+)DDfGXddni%)=jG7ToI_1P}o}%pI_&pO!zMpUJItFYk_v z9-kdRD{vUVyjQry5rR8#;SRGPCMG8NbfW+^=mDr#*BgOc~P$Q}KL4Mp4r!UPqsGf@qs` z7B;iJMD+$^W*23&ADyz1CmT}eo|G?&hrXsp{LNta~KAVHpgTw}5zib-TBCP}<5KE5_jJlx-K z)ULhn$p6Xw(4fF;0w4YX%TNO;tX9E%BfN~wH*GPnn^#lFH`yr}S!f+;%0{>(xz(Jl zF47Dhz=PxOS4>8?V8wfI6-4|T*Dj}`lDTHoqMA$}X+h7ipx&HCLZh^FGyWc!-m?4( zjDy>AS5n+Ivp|9KG(cKCgGpskn@b&1iFMzHbf{+@ zonO>y+j?im>)RyKLwl#dHf~^+^WYpF6u8T)qAgQ9Oc55{y|m%SI)2>Dug}sFQr@a( zInAAh=Nf~!&tFHMhKbXkzp}S5ZC50bK7m8$kP-&G_F`9*J7j3|Bj%Xb!`*W$raawy zcrQ~WOYhumZ%%Kw5Pu$Z%@@zSww*zGZ)F0#bej4wTq-?|P7HR=T*(-3opS0kQHRs2 z!}_e`G1c+Y)bg3{mhcpITWHH(QYNWcp|0scvJ3=R2)c1-Cs%%7Ev^36l$mT)&~)R z?oE38J6g?f;yOugMnP^7zRc&k{yoaAi%Q1*b)unU)X78cFJGizkXDB(8o~9W>#_lJ z74yN$VbflZ25{RaX@gIbhdyaSjAI?gJCYwf8UvjGUgCvBgwm};5M$F+oa*DZ@!I0U zJvwNIM7QEI%e<90Ui$Qc_a|PBaO>b(m; zz8O0q6==O=AeY4qtt=pZGV@r%cY;O%jI|#)Gh87pe2aX;<2SV8=O|ynr-9#99vMUK})1b?upAW0QEcfE*w?+DgS?KE);Ac59ic+{cC5hul zzYiT04 z7Q~+~9&SH4ck?L&k4k|%uAoRYgjth)Er))6&)|t!d>?ykYMW$80~C^(41j2R2%1A= zz>qk!Ya-f1l|;S+>xn9ZL)}S!?1h!ERl1F}sdh zW!KCj0ogV=*g5v2qYEqTzZWl;DO|4K*W8#N)G@JzYKQSfQ2Pudose~jnR5_(Hg3GG zMP*ZX1EjHG4xAAq$HkcOCD0l_&k=5iM0A&$kYl;o)x&1a7Y0^aU-Xe6BzZT^ud_Ay zwC0Y=LPmTGq&_|zCqBzBgIujQKh5< z*9J;GJv@p_81DwdP^uv(5A4n!k}7oVcvE4gL!nlh@z;q0&OixEgpSWSiq?Ebn$RA$ zkRMnUJylV#!TnzUP5uaqS9SQB0gXGhSV`Mb^mgdBVrd(6RpB+fCOwJkI`i6juvE#@ zQ+``^QB0ppPVq5Z z?W5;H65)@|be!0`tpxLFW~!*JotyKNOUahvq?e+z(c!JyM^sz-svzgrfP6bG)80nU z7BS5|2#`0@no>4viJIkg7DS6v2oFwJ&nXu35FnwPK832@4kz*h8C|{aT2*D>Q%&H9 z?fmlBA|aKi##;OsFv*-vOtx22P#VixU%ThYa?eug zlxq7l>LJ3jdKai|dvIP+#b7hGetFZw`I@no^s-Re4EJG1{|Nq>1$j&4rVX-*0XI1f ze3g_Kn3d~fVWG$cnJ*K;x?tS>OMXqzUiboGHxB|2c~|hpbiYk4hz#UCl+a5q$jCN} zWHSo(aAXsKSZeaHb#@^(GiL)U*k$Agx1m<&e!=Stz*ZMvzw6 zF*j+oGG}WA^h9ega20~eQQ@Lxm+LXqFcQY9m<8MLBs*YX?Z=X;Ow$yjfi;fWQY4}$ zM)6OUL7(){U+Bw2UvCenCequfnqE}Iw2;8;Gx+%FK@5)vi{wO6NVf1eb~-!4LWCP@ zDZ-8QWv!S(M5qR_tuML%~BcHXrXP< zkQh0A6%yy3Bk{7i92vT++>zEeqkbw%Sr(*_N9Odn{UXGSs*CVnZHkfwd{WoHJm{ur zc#fP$7p`Oc-25tTd*yw9zYsA28sqS@PnF?G@h^U=tc-YeN}~n8B9U9z zpbz!}yPm1dO`gR8`-#lrCx+pPgz*Z1!P*1Er4uqmJHVT&A30H5bIl_UT;>O)4gi+b zL*BpyK%GwQ(>-k3$*NZR(Nn@l?knuKhswWzWw?b=ka*FnXAsy4WE0kuiN4u4izhj& zNb-O`Bk}+)S@{swG6&&XI?#*3Q0+anb>$T*hsziSVH&OJokZ=gmX0E2^z&ydmosH( zYWW3{TlTH~PD;PEK?+U+x_rud={x0+cg5q}Ze!6um2WNR^=OZ9cp&=J1^tvtB1TzmNOL(Bd+=LoyDHw6(f3O)XO&ZxO+3>iZFJ z!)X_DE&{d5V$MH?(^-mindudK>47MwwMxdhpEreWk3iROzLSS$Na&Y%1FyLpmFUn8 zIuD_2;MBZFGr3?6GykaHSO#p&9N@`@2l$>hr;dI4`E)^tckYcftb#LK6Per2IroZ! zuH=j^L=qPV#SZU?zZs2uPF;%nV~xVPEpOdUv!ZE`w{T02*H(NP1`ym|e@AwQcE`rA z+2Ek`gZm0}2i}hh;4Idk4k*;04zN+94UZj4Zy)cKvA^c4p{i&A7pNO*_Sc@ANKPmM zFOV0o8}bvW9n2Hc&WwK#fEn;T&5r#Jq#uk+8r~TNfM%pEq#V8%W}Zi`$WtoeM<;}? z@RM4vtN$h7Hnq->H0(vO}U8$By0W{1; zZ|^h6Kupfmu;^j{J@lxk7jN7|SFhn$2?L2K+ow>)<_!|6`3@Z`TyBCJxH56fCeqlj zx*F$bsZPLj$0d>bnXL|8yKEN@1BitFbBqJkmmn%aJGE%#@aD;@4n2EnK}gnIVbSW& z@@*9tg>~*0ITB?woWs5XT8!teO!+&DtUq(Sq#_bT&xEqv>^<@fGOv+=G_pMnKl_v@ytmJgR? z$I;fqqkPLtx9_D&WqTbEcov|FykGT{*2~m{UEJfl7orO4$+)?_viBa}rds5E7u}ks zn`B@@O%tgOAz`WsaCjDUHc!r6_qF~gQ7^^kb1wv~cVq)nLnZp!iFJ(TjGD&U#VGk! zs-4HJZ5w@)a(?3OJH}!b7uQ$LuG<$HY9ioNk2@|No|Yb4{S!SFlQ{KE&)AXe*uqCp z+XXWA*O9qTa1PXz6u=(1HgC>fnk#2y&?Fgr?F(YJ&z;^u?5FQh5|MlF7Evojxd<%2E;FLr`Gpjv)@qa>7(Z{ z_)(=^)Olp-vbZ=N{d-3TkW?R|iksw^Spjy*i|q83n+SXIT#4C_EDP}FjO{+4!v-|iVxbAD`I~R}! zQM;Pzs%;99SWS#ifBhO(Tn-2cdF!A7&qTH?RxGEC#tAG<3PHfzSnKO*g`zr~!gvt_ z?l2ggWt7ulzY8j zOz#$|&)Lv2Sn-cMVLub6;B9NPUiWtYwm=!$cy;jc3$uFjRF2j{_Q%*`%y&7qdq-AV z@dj%NPv7{dyFwB3K%z*NrWuD%k7lE=%HUI7pRdeFAaQK?R~jao+Kffz8hwkWxe+gI zhMGoEqQO-m+{Jd|4Ogw@^Km}yT%`g{$tLBz*%}(+u*MDFjnV$TeU6n`F))5K@Hhv6 zR3fnJWZP+&tOkp1GPB3f3z~r!x83RadPud0;lhBc&+V_C&%=9L@eQdW_lQR7wM#mf zNxh%bQqI==KTZTyNTnJgm``wL%4#ZApZb49@%H(f(7vUn)i4jRM$ko9T1dwcwxE8G zB_;LrDk@9lhKra-N|Z;a0UR1L(myo;=~~zfX4nS=9UwQQ-ZJVfWn>X=fIE(k0SF5A zgM&GW!u%xC>~dDahsOQ^^Mc88;wYBOY4B^-=_cfEtZ*i~TJ=g$UE4xrM@Rd&g&F>X z1R{Q*R)g4RG_hn3&+l7L&?<3pMI169gBvL9{P{vAbAkM2>FJ`h#T}`JBFz>spj}m( z81wV3T_?7rcGp%3Y+`4dQv_FCL~OJ7s^}3k&??nC(hy(eMf%03BZ!D5Ruq<7?l zsYx#rolD{q7;<>5(*^W-=sjS}QxiXwg!oXFU>!?gop)i~yE0HN*Gt~yARV_43foN8 zP37DYJwp|a`2lDFI510iq37N+jyd<+c<>R)^!;szFqMe!@_c?zoXRrROL`6|`21L{ zivnu;IWpM;3p5V+Y3|@VudJ^0Z=G2c@&1~jAX15yquA4ziSEg>pC{^dwLMQYerDp* zOCRol?!Jt3vpKl?_HCt>zSFG*61;ET)xL!loVK|WnkpJU)-0vQC$tYq4xLdR5JNRp zRrkv4+ZQD>yXq@9n(){#zWd8&tv!ZR?5nj^K(p?k(Y}RGPgt%9F64qVbSs2nA$B(GYrE{4NQPu(xM zKRD29D4f`}XBpF4a)+IhiaFVzsx(7og3X?0laFcqYVN=`j)7V_FC$)@%Z`@5 zjm84Xuie9i@bv8H6ficlb=@hEzBSXTT6!HnGiFsYZqwYWtWcDyR5jHO-_+JoEhuj) zZy2{wS&CSDx6FU|IV=T1a;wekbG9$yf1OaN8ks!jJ1yNnIv`vgr`x1LL#O)Wt5Pu$ zkqqT2Qs)4yZJze?_VcR_{@>{e`Vx@v;a@d$gfF@R{1;up$ky*KId_!+hGywStR%s!I9FNnMk0Nwpz-ULq1bRUFyV&+eqre{)tfUKEgA*Lrqx9FZ z?CbuvoQ~6MI_YvRWS%%3M*Z=@O7{c7re5M5Zk7-_%kI3&>pzXkoOY|W4!-L)usMfcuSfE$%Lx9TA9@V+ ztqg_C^v!Mlg0E4+x+E|Ie8?r}Ho;`qREY&o?Wzv@G4wHT&{<!-Ll>N_xoQ6!nCAO!$bU-(@FY#-*xYQvcEE> zbA)BU*N6N_I@B1qBD*2A-4o33#x}SqS+wLF?<8h4%n6Hdby#=iIB932s(4f8cgJf%bL{#%PiCs`+w__Ao*uPH(JcZ)W(buj*4lm42s z|GPyc^vMn2qYqt@e!#&#rnoFvCp1efo*g8R{50z%@H!2nf!A|+y(~Ugwfxl=1l>6j z``8^v3+IgDCLxtJJv+@IL*Hx#&e9a`Do$sU{Nu;1mXd+YDj5x++@4#ArCpMem_0PiJ-;v+ zu_>_PrOXl_`+&Rv?7T*nFyzNICw(b5`pZdM1t!$$bOhkI(li7Ue>UTD$VEjSefcweKx*7x{LksSf}d6Iqb)#B@7MD_ z=GS-C(w`t#y(2{4=?yoW-fq;YfdOme+vF5`R?BZe}A>'Credits & Adjustments' is null then 'null' else '' end <> 'null' +) +TO 'C:\users\ptrowbridge\downloads\pncl.csv' WITH (format csv, header TRUE) + +--source +SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCL' + + diff --git a/database/deploy/reload/pncl/load.cmd b/database/deploy/reload/pncl/load.cmd new file mode 100644 index 0000000..19fdc66 --- /dev/null +++ b/database/deploy/reload/pncl/load.cmd @@ -0,0 +1,2 @@ +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set +curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pncl.csv http://localhost:81/import?srce=PNCL \ No newline at end of file diff --git a/database/deploy/reload/pncl/srce.json b/database/deploy/reload/pncl/srce.json new file mode 100644 index 0000000..fffd0de --- /dev/null +++ b/database/deploy/reload/pncl/srce.json @@ -0,0 +1,73 @@ +{ + "constraint": [ + "{PostDate}", + "{Schedule#}" + ], + "source": "client_file", + "loading_function": "csv", + "name": "PNCL", + "schemas": { + "default": [ + { + "path": "{Schedule#}", + "type": "text", + "column_name": "Schedule#" + }, + { + "type": "date", + "column_name": "PostDate", + "path": "{PostDate}" + }, + { + "type": "text", + "column_name": "Assn#", + "path": "{Assn#}" + }, + { + "type": "text", + "column_name": "Coll#", + "path": "{Coll#}" + }, + { + "type": "numeric", + "column_name": "AdvanceRate", + "path": "{AdvanceRate}" + }, + { + "type": "numeric", + "column_name": "Sales", + "path": "{Sales}" + }, + { + "type": "numeric", + "column_name": "Credits & Adjustments", + "path": "{Credits & Adjustments}" + }, + { + "type": "numeric", + "column_name": "Gross Collections", + "path": "{Gross Collections}" + }, + { + "type": "numeric", + "column_name": "CollateralBalance", + "path": "{CollateralBalance}" + }, + { + "type": "numeric", + "column_name": "MaxEligible", + "path": "{MaxEligible}" + }, + { + "type": "numeric", + "column_name": "Ineligible Amount", + "path": "{Ineligible Amount}" + }, + { + "type": "numeric", + "column_name": "Reserve Amount", + "path": "{Reserve Amount}" + } + ] + } +} \ No newline at end of file diff --git a/database/deploy/reload/pnco/extract.sql b/database/deploy/reload/pnco/extract.sql new file mode 100644 index 0000000..fee80bf --- /dev/null +++ b/database/deploy/reload/pnco/extract.sql @@ -0,0 +1,32 @@ +--transactions with date in download format for constraint +--transactions with date in download format for constraint +COPY +( +SELECT + r."Loan#" + ,to_char(r."Post Date",'mm/dd/yyyy') "Post Date" + ,to_char(r."Effective Date",'mm/dd/yyyy') "Effective Date" + ,r."Reference #" + ,r."Description" + ,r."Advances" + ,r."Adjustments" + ,r."Payments" + ,r."Loan Balance" +FROM + tps.trans + JOIN LATERAL jsonb_populate_record(NULL::tps.pnco, rec) r ON TRUE +WHERE + srce = 'PNCO' +) +TO 'C:\users\ptrowbridge\downloads\pnco.csv' WITH (format csv, header TRUE) + +--source +SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCO' + +--mapdef +SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PNCO') x + +--map values +SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PNCO') X + + diff --git a/database/deploy/reload/pnco/load.cmd b/database/deploy/reload/pnco/load.cmd new file mode 100644 index 0000000..b2a09e4 --- /dev/null +++ b/database/deploy/reload/pnco/load.cmd @@ -0,0 +1,2 @@ +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set +curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pnco.csv http://localhost:81/import?srce=PNCO \ No newline at end of file diff --git a/database/deploy/reload/pnco/srce.json b/database/deploy/reload/pnco/srce.json new file mode 100644 index 0000000..28b948a --- /dev/null +++ b/database/deploy/reload/pnco/srce.json @@ -0,0 +1,60 @@ +{ + "name": "PNCO", + "source": "client_file", + "loading_function": "csv", + "constraint": [ + "{Post Date}", + "{Effective Date}", + "{Loan#}", + "{Reference #}" + ], + "schemas": { + "default": [ + { + "path": "{Loan#}", + "type": "text", + "column_name":"Loan#" + }, + { + "path": "{Post Date}", + "type": "date", + "column_name":"Post Date" + }, + { + "path": "{Effective Date}", + "type": "date", + "column_name":"Effective Date" + }, + { + "path": "{Reference #}", + "type": "text", + "column_name":"Reference #" + }, + { + "path": "{Description}", + "type": "text", + "column_name":"Description" + }, + { + "path": "{Advances}", + "type": "numeric", + "column_name":"Advances" + }, + { + "path": "{Adjustments}", + "type": "numeric", + "column_name":"Adjustments" + }, + { + "path": "{Payments}", + "type": "numeric", + "column_name":"Payments" + }, + { + "path": "{Loan Balance}", + "type": "numeric", + "column_name":"Loan Balance" + } + ] + } +} \ No newline at end of file diff --git a/database/deploy/reload/wmpd/curl_hunt.cmd b/database/deploy/reload/wmpd/curl_hunt.cmd new file mode 100644 index 0000000..16bf611 --- /dev/null +++ b/database/deploy/reload/wmpd/curl_hunt.cmd @@ -0,0 +1,3 @@ +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set +curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set +curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/WMPD.csv http://localhost:81/import?srce=WMPD \ No newline at end of file diff --git a/database/deploy/reload/wmpd/extract.sql b/database/deploy/reload/wmpd/extract.sql new file mode 100644 index 0000000..9113bc6 --- /dev/null +++ b/database/deploy/reload/wmpd/extract.sql @@ -0,0 +1,63 @@ +--source +COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'WMPD') TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\srce.json' WITH (FORMAT TEXT, HEADER FALSE) + +--mapdef +COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'WMPD') x) TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\map.json' WITH (FORMAT TEXT, HEADER FALSE) + +--map values +SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'WMPD') X + +--records +copy ( + select + r."Carrier", + r."SCAC", + r."Mode", + r."Pro #", + r."B/L", + r."Pd Amt", + r."Loc#", + r."Pcs", + r."Wgt", + r."Chk#", + r."Pay Dt", + r."Acct #", + r."I/O", + r."Sh Nm", + r."Sh City", + r."Sh St", + r."Sh Zip", + r."Cons Nm", + r."D City ", + r."D St", + r."D Zip", + r."Sh Dt", + r."Inv Dt", + r."Customs Entry#", + r."Miles", + r."Frt Class", + r."Master B/L" + from + tps.trans + join lateral jsonb_populate_record(null::tps.WMPD, rec) r on true + where + srce = 'WMPD' + order by + r."Pay Dt" asc +) to +'C:\users\ptrowbridge\downloads\WMPD.csv' with (format csv, header true); + +--rebuild source def to include PATH +SELECT + ae.r + ||jsonb_build_object( + 'path', + ( + '{'||(ae.r->>'column_name')||'}' + ) + ) +FROM + tps.srce + JOIN LATERAL jsonb_array_elements(defn->'schemas'->'default') ae(r) ON TRUE +WHERE + srce = 'WMPD' diff --git a/database/deploy/reload/wmpd/srce.json b/database/deploy/reload/wmpd/srce.json new file mode 100644 index 0000000..eaef776 --- /dev/null +++ b/database/deploy/reload/wmpd/srce.json @@ -0,0 +1,148 @@ +{ + "name": "WMPD", + "source": "client_file", + "loading_function": "csv", + "constraint": [ + "{Pay Dt}", + "{Carrier}" + ], + "schemas": { + "default": [ + { + "path": "{Carrier}", + "type": "text", + "column_name": "Carrier" + }, + { + "path": "{SCAC}", + "type": "text", + "column_name": "SCAC" + }, + { + "path": "{Mode}", + "type": "text", + "column_name": "Mode" + }, + { + "path": "{Pro #}", + "type": "text", + "column_name": "Pro #" + }, + { + "path": "{B/L}", + "type": "text", + "column_name": "B/L" + }, + { + "path": "{Pd Amt}", + "type": "numeric", + "column_name": "Pd Amt" + }, + { + "path": "{Loc#}", + "type": "text", + "column_name": "Loc#" + }, + { + "path": "{Pcs}", + "type": "numeric", + "column_name": "Pcs" + }, + { + "path": "{Wgt}", + "type": "numeric", + "column_name": "Wgt" + }, + { + "path": "{Chk#}", + "type": "numeric", + "column_name": "Chk#" + }, + { + "path": "{Pay Dt}", + "type": "date", + "column_name": "Pay Dt" + }, + { + "path": "{Acct #}", + "type": "text", + "column_name": "Acct #" + }, + { + "path": "{I/O}", + "type": "text", + "column_name": "I/O" + }, + { + "path": "{Sh Nm}", + "type": "text", + "column_name": "Sh Nm" + }, + { + "path": "{Sh City}", + "type": "text", + "column_name": "Sh City" + }, + { + "path": "{Sh St}", + "type": "text", + "column_name": "Sh St" + }, + { + "path": "{Sh Zip}", + "type": "text", + "column_name": "Sh Zip" + }, + { + "path": "{Cons Nm}", + "type": "text", + "column_name": "Cons Nm" + }, + { + "path": "{D City }", + "type": "text", + "column_name": "D City " + }, + { + "path": "{D St}", + "type": "text", + "column_name": "D St" + }, + { + "path": "{D Zip}", + "type": "text", + "column_name": "D Zip" + }, + { + "path": "{Sh Dt}", + "type": "date", + "column_name": "Sh Dt" + }, + { + "path": "{Inv Dt}", + "type": "date", + "column_name": "Inv Dt" + }, + { + "path": "{Customs Entry#}", + "type": "text", + "column_name": "Customs Entry#" + }, + { + "path": "{Miles}", + "type": "numeric", + "column_name": "Miles" + }, + { + "path": "{Frt Class}", + "type": "text", + "column_name": "Frt Class" + }, + { + "path": "{Master B/L}", + "type": "text", + "column_name": "Master B/L" + } + ] + } +} \ No newline at end of file diff --git a/database/deploy/setup.sql b/database/deploy/setup.sql new file mode 100644 index 0000000..e184a2e --- /dev/null +++ b/database/deploy/setup.sql @@ -0,0 +1,2150 @@ +------create dev schema and api user----------------------------------------------------------------------------------------------------------------- + +DROP SCHEMA IF EXISTS tps CASCADE; +DROP SCHEMA IF EXISTS tpsv CASCADE; + +CREATE SCHEMA tps; +COMMENT ON SCHEMA tps IS 'third party source data'; + +CREATE SCHEMA tpsv; +COMMENT ON SCHEMA tps IS 'third party source views'; + +DROP USER IF EXISTS api; + +CREATE ROLE api WITH + LOGIN + NOSUPERUSER + NOCREATEDB + NOCREATEROLE + INHERIT + NOREPLICATION + CONNECTION LIMIT -1 + ENCRYPTED PASSWORD 'md56da13b696f737097e0146e47cc0d0985'; + +-----need to setup all database objects and then grant priveledges to api---------------------------------------------------------------------------- + +--grant schema USAGE +GRANT USAGE ON SCHEMA tps TO api; +GRANT USAGE ON SCHEMA tpsv TO api; + +--grant current table privledges +GRANT SELECT, UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA tps TO api; +GRANT SELECT, UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA tpsv TO api; + +--grant current sequence privledges +GRANT USAGE ON ALL SEQUENCES IN SCHEMA tps TO api; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA tpsv TO api; + +--grant future table privledges +ALTER DEFAULT PRIVILEGES IN SCHEMA tps GRANT SELECT, UPDATE, INSERT, DELETE ON TABLES TO api; +ALTER DEFAULT PRIVILEGES IN SCHEMA tpsv GRANT SELECT, UPDATE, INSERT, DELETE ON TABLES TO api; + +--grant future sequence privledges +ALTER DEFAULT PRIVILEGES IN SCHEMA tps GRANT USAGE ON SEQUENCES TO api; +ALTER DEFAULT PRIVILEGES IN SCHEMA tpsv GRANT USAGE ON SEQUENCES TO api; + + +-----create tables----------------------------------------------------------------------------------------------------------------------------------- + +-----regex map instructions table + +CREATE TABLE tps.map_rm ( + srce text NOT NULL, + target text NOT NULL, + regex jsonb, + seq integer NOT NULL, + hist jsonb +); +COMMENT ON TABLE tps.map_rm IS 'regex map instructions'; + +-----return value table + +CREATE TABLE tps.map_rv ( + srce text NOT NULL, + target text NOT NULL, + retval jsonb NOT NULL, + map jsonb NOT NULL, + hist jsonb NOT NULL +); +COMMENT ON TABLE tps.map_rv IS 'return value lookup table'; + +-----source definition table + +CREATE TABLE tps.srce ( + srce text NOT NULL, + defn jsonb, + hist jsonb +); +COMMENT ON TABLE tps.srce IS 'source master listing and definition'; + +-----source data table + +CREATE TABLE tps.trans ( + id integer NOT NULL, + srce text, + rec jsonb, + parse jsonb, + map jsonb, + allj jsonb, + ic jsonb, + logid INTEGER +); +COMMENT ON TABLE tps.trans IS 'source records'; +COMMENT ON COLUMN tps.trans.ic IS 'input constraint value'; +ALTER TABLE tps.trans ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME tps.trans_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + +-----import log table + +CREATE TABLE tps.trans_log ( + id integer NOT NULL, + info jsonb +); +COMMENT ON TABLE tps.trans_log IS 'import event information'; +ALTER TABLE tps.trans_log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME tps.trans_log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-------------primary keys---------------------------------------------------------------------------------------------------------------------------- + +ALTER TABLE ONLY tps.map_rm + ADD CONSTRAINT map_rm_pk PRIMARY KEY (srce, target); + +ALTER TABLE ONLY tps.map_rv + ADD CONSTRAINT map_rv_pk PRIMARY KEY (srce, target, retval); + +ALTER TABLE ONLY tps.srce + ADD CONSTRAINT srce_pkey PRIMARY KEY (srce); + +ALTER TABLE ONLY tps.trans_log + ADD CONSTRAINT trans_log_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY tps.trans + ADD CONSTRAINT trans_pkey PRIMARY KEY (id); + +-------------indexes--------------------------------------------------------------------------------------------------------------------------------- + +CREATE INDEX trans_allj ON tps.trans USING gin (allj); + +CREATE INDEX trans_rec ON tps.trans USING gin (rec); + +CREATE INDEX trans_srce ON tps.trans USING btree (srce); + +-------------foreign keys---------------------------------------------------------------------------------------------------------------------------- + +ALTER TABLE ONLY tps.map_rm + ADD CONSTRAINT map_rm_fk_srce FOREIGN KEY (srce) REFERENCES tps.srce(srce); + +ALTER TABLE ONLY tps.map_rv + ADD CONSTRAINT map_rv_fk_rm FOREIGN KEY (srce, target) REFERENCES tps.map_rm(srce, target); + +ALTER TABLE ONLY tps.trans + ADD CONSTRAINT trans_srce_fkey FOREIGN KEY (srce) REFERENCES tps.srce(srce); + +ALTER TABLE ONLY tps.trans + ADD CONSTRAINT trans_logid_fkey FOREIGN KEY (logid) REFERENCES tps.trans_log(id); + +-------------create functions------------------------------------------------------------------------------------------------------------------------ + +-----set source +DROP FUNCTION IF EXISTS tps.srce_set(jsonb); +CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + ---------test if anythign is changing-------------------------------------------------------------------------------------------- + + IF _defn = (SELECT defn FROM tps.srce WHERE srce = _defn->>'name') THEN + _message:= + ( + $$ + { + "status":"complete", + "message":"source was not different no action taken" + } + $$::jsonb + ); + RETURN _message; + END IF; + + ---------if the constraint definition is changing, rebuild for existing records--------------------------------------------------- + + SELECT + NOT (_defn->'constraint' = (SELECT defn->'constraint' FROM tps.srce WHERE srce = _defn->>'name')) + INTO + _rebuild; + + RAISE NOTICE '%',_rebuild::text; + + ---------do merge----------------------------------------------------------------------------------------------------------------- + + INSERT INTO + tps.srce (srce, defn, hist) + SELECT + --extract name from defintion + _defn->>'name' + --add current timestamp to defintions + ,_defn + --add definition + ,jsonb_build_object( + 'hist_defn',_defn + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + ON CONFLICT ON CONSTRAINT srce_pkey DO UPDATE + SET + defn = _defn + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',_defn + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + srce.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + --rebuild constraint key if necessary--------------------------------------------------------------------------------------- + + IF _rebuild THEN + WITH + rebuild AS ( + SELECT + j.srce + ,j.rec + ,j.id + --aggregate back to the record since multiple paths may be listed in the constraint + ,tps.jsonb_concat_obj( + jsonb_build_object( + --the new json key is the path itself + cons.path->>0 + ,j.rec#>((cons.path->>0)::text[]) + ) + ) json_key + FROM + tps.trans j + INNER JOIN tps.srce s ON + s.srce = j.srce + JOIN LATERAL jsonb_array_elements(s.defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE + WHERE + s.srce = _defn->>'name' + GROUP BY + j.rec + ,j.id + ) + UPDATE + tps.trans t + SET + ic = r.json_key + FROM + rebuild r + WHERE + t.id = r.id; + _message:= + ( + $$ + { + "status":"complete", + "message":"source set and constraint rebuilt on existing records" + } + $$::jsonb + ); + ELSE + _message:= + ( + $$ + { + "status":"complete", + "message":"source set" + } + $$::jsonb + ); + END IF; + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error importing data" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql; + +-----generate sql to create select based on schema +DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text); +CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT +AS +$f$ +DECLARE + --_schema text; + --_srce text; + _sql text; +BEGIN + --_schema:= 'default'; + --_srce:= 'dcard'; +SELECT + 'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';' +INTO + _sql +FROM + tps.srce s + JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE + JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE + JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE +WHERE + srce = _srce + AND list.e->>'name' = _schema +GROUP BY + s.srce + ,list.e; + +RETURN _sql; +RAISE NOTICE '%',_sql; + +END +$f$ +LANGUAGE plpgsql; + +-----set map defintion from json argument +CREATE OR REPLACE FUNCTION tps.srce_map_def_set(_defn jsonb) RETURNS jsonb +AS +$f$ + +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + BEGIN + + INSERT INTO + tps.map_rm (srce, target, regex, seq, hist) + SELECT + --data source + ae.r->>'srce' + --map name + ,ae.r->>'name' + --map definition + ,ae.r + --map aggregation sequence + ,(ae.r->>'sequence')::INTEGER + --history definition + ,jsonb_build_object( + 'hist_defn',ae.r + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + FROM + jsonb_array_elements(_defn) ae(r) + ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET + srce = excluded.srce + ,target = excluded.target + ,regex = excluded.regex + ,seq = excluded.seq + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.regex + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rm.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + + EXCEPTION WHEN OTHERS THEN + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting definition" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; + END; + + _message:= jsonb_build_object('status','complete','message','definition has been set'); + return _message; + +END; +$f$ +language plpgsql; + + +------------build report for unmapped items--------------------------------------------------------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION tps.jsonb_concat( + state jsonb, + concat jsonb) + RETURNS jsonb AS +$BODY$ +BEGIN + --RAISE notice 'state is %', state; + --RAISE notice 'concat is %', concat; + RETURN state || concat; +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; + +DROP AGGREGATE IF EXISTS tps.jsonb_concat_obj(jsonb); +CREATE AGGREGATE tps.jsonb_concat_obj(jsonb) ( + SFUNC=tps.jsonb_concat, + STYPE=jsonb, + INITCOND='{}' +); + + +DROP FUNCTION IF EXISTS tps.report_unmapped; +CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint +) +LANGUAGE plpgsql +AS +$f$ +BEGIN + +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; +END; +$f$; + + +-------------------create trigger to map imported items------------------------------------------------------------------------------------------------------ + +CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER +AS +$f$ + DECLARE + _cnt INTEGER; + + BEGIN + IF (TG_OP = 'INSERT') THEN + + + WITH + --------------------apply regex operations to transactions----------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN new_table t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT count(*) FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ORDER BY + id + ) + + + --SELECT * FROM agg_to_target + + + , link_map AS ( + SELECT + a.srce + ,a.id + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a.retain_val retain_value + ,v.map + FROM + agg_to_target a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val + ) + + --SELECT * FROM link_map + + , agg_to_id AS ( + SELECT + srce + ,id + ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val + ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map + FROM + link_map + GROUP BY + srce + ,id + ) + + --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100 + + --create a complete list of all new inserts assuming some do not have maps (left join) + ,join_all AS ( + SELECT + n.srce + ,n.id + ,n.rec + ,a.retain_val parse + ,a.map + ,n.rec||COALESCE(a.map||a.retain_val,'{}'::jsonb) allj + FROM + new_table n + LEFT OUTER JOIN agg_to_id a ON + a.id = n.id + ) + + --update trans with join_all recs + UPDATE + tps.trans t + SET + parse = a.parse + ,map = a.map + ,allj = a.allj + FROM + join_all a + WHERE + t.id = a.id; + + + END IF; + RETURN NULL; + END; +$f$ LANGUAGE plpgsql; + +CREATE TRIGGER trans_insert + AFTER INSERT ON tps.trans + REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE tps.trans_insert_map(); + + +------------------import data------------------------------------------------------------------------------------------------------------------------------------- + +DROP FUNCTION IF EXISTS tps.srce_import(text, jsonb); +CREATE OR REPLACE FUNCTION tps.srce_import(_srce text, _recs jsonb) RETURNS jsonb + +/*-------------------------------------------------------- +0. test if source exists +1. create pending list +2. get unqiue pending keys +3. see which keys not already in tps.trans +4. insert pending records associated with keys that are not already in trans +5. insert summary to log table +*/--------------------------------------------------------- + +--to-do +--return infomation to a client via json or composite type + + +AS $f$ +DECLARE + _t text; + _c text; + _log_info jsonb; + _log_id text; + _cnt numeric; + _message jsonb; + --_recs jsonb; + --_srce text; + _defn jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + --_path := 'C:\users\fleet\downloads\discover-recentactivity-20171031.csv'; + --_srce := 'dcard'; + --_recs:= $$[{"Trans. Date":"1/2/2018","Post Date":"1/2/2018","Description":"GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV","Amount":4.26,"Category":"Services"},{"Trans. Date":"1/2/2018","Post Date":"1/2/2018","Description":"MICROSOFT *ONEDRIVE 800-642-7676 WA","Amount":4.26,"Category":"Services"},{"Trans. Date":"1/3/2018","Post Date":"1/3/2018","Description":"CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3","Amount":200,"Category":"Medical Services"},{"Trans. Date":"1/4/2018","Post Date":"1/4/2018","Description":"AT&T *PAYMENT 800-288-2020 TX","Amount":57.14,"Category":"Services"},{"Trans. Date":"1/4/2018","Post Date":"1/7/2018","Description":"WWW.KOHLS.COM #0873 MIDDLETOWN OH","Amount":-7.9,"Category":"Payments and Credits"},{"Trans. Date":"1/5/2018","Post Date":"1/7/2018","Description":"PIZZA HUT 007946 STOW OH","Amount":9.24,"Category":"Restaurants"},{"Trans. Date":"1/5/2018","Post Date":"1/7/2018","Description":"SUBWAY 00044289255 STOW OH","Amount":10.25,"Category":"Restaurants"},{"Trans. Date":"1/6/2018","Post Date":"1/7/2018","Description":"ACME NO. 17 STOW OH","Amount":103.98,"Category":"Supermarkets"},{"Trans. Date":"1/6/2018","Post Date":"1/7/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":1.69,"Category":"Merchandise"},{"Trans. Date":"1/6/2018","Post Date":"1/7/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":2.19,"Category":"Merchandise"},{"Trans. Date":"1/9/2018","Post Date":"1/9/2018","Description":"CIRCLE K 05416 STOW OH00947R","Amount":3.94,"Category":"Gasoline"},{"Trans. Date":"1/9/2018","Post Date":"1/9/2018","Description":"CIRCLE K 05416 STOW OH00915R","Amount":52.99,"Category":"Gasoline"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"AUTOZONE #0722 STOW OH","Amount":85.36,"Category":"Automotive"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":26.68,"Category":"Merchandise"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"EL CAMPESINO STOW OH","Amount":6.5,"Category":"Restaurants"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"TARGET STOW OH","Amount":197.9,"Category":"Merchandise"},{"Trans. Date":"1/14/2018","Post Date":"1/14/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":13.48,"Category":"Merchandise"},{"Trans. Date":"1/15/2018","Post Date":"1/15/2018","Description":"TARGET.COM * 800-591-3869 MN","Amount":22.41,"Category":"Merchandise"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"BUFFALO WILD WINGS KENT KENT OH","Amount":63.22,"Category":"Restaurants"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"PARTA - KCG KENT OH","Amount":4,"Category":"Government Services"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"REMEMBERNHU 402-935-7733 IA","Amount":60,"Category":"Services"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"TARGET.COM * 800-591-3869 MN","Amount":44.81,"Category":"Merchandise"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"TREE CITY COFFEE & PASTR KENT OH","Amount":17.75,"Category":"Restaurants"},{"Trans. Date":"1/17/2018","Post Date":"1/17/2018","Description":"BESTBUYCOM805526794885 888-BESTBUY MN","Amount":343.72,"Category":"Merchandise"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":5.98,"Category":"Merchandise"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"U-HAUL OF KENT-STOW KENT OH","Amount":15.88,"Category":"Travel/ Entertainment"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"WALMART GROCERY 800-966-6546 AR","Amount":5.99,"Category":"Supermarkets"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"WALMART GROCERY 800-966-6546 AR","Amount":17.16,"Category":"Supermarkets"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"WALMART GROCERY 800-966-6546 AR","Amount":500.97,"Category":"Supermarkets"},{"Trans. Date":"1/20/2018","Post Date":"1/20/2018","Description":"GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W","Amount":2.12,"Category":"Services"},{"Trans. Date":"1/20/2018","Post Date":"1/20/2018","Description":"LOWE'S OF STOW, OH. STOW OH","Amount":256.48,"Category":"Home Improvement"},{"Trans. Date":"1/23/2018","Post Date":"1/23/2018","Description":"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT","Amount":-32.2,"Category":"Awards and Rebate Credits"},{"Trans. Date":"1/23/2018","Post Date":"1/23/2018","Description":"INTERNET PAYMENT - THANK YOU","Amount":-2394.51,"Category":"Payments and Credits"},{"Trans. Date":"1/27/2018","Post Date":"1/27/2018","Description":"GIANT-EAGLE #4096 STOW OH","Amount":67.81,"Category":"Supermarkets"},{"Trans. Date":"1/27/2018","Post Date":"1/27/2018","Description":"OFFICEMAX/OFFICE DEPOT63 STOW OH","Amount":21.06,"Category":"Merchandise"},{"Trans. Date":"1/27/2018","Post Date":"1/27/2018","Description":"TARGET STOW OH","Amount":71,"Category":"Merchandise"},{"Trans. Date":"1/29/2018","Post Date":"1/29/2018","Description":"NETFLIX.COM NETFLIX.COM CA19899514437","Amount":14.93,"Category":"Services"},{"Trans. Date":"1/30/2018","Post Date":"1/30/2018","Description":"SQ *TWISTED MELTZ KENT OH0002305843011416898511","Amount":16.87,"Category":"Restaurants"},{"Trans. Date":"1/30/2018","Post Date":"1/30/2018","Description":"TARGET STOW OH","Amount":49.37,"Category":"Merchandise"}]$$::jsonb; + +----------------------------------------------------test if source exists---------------------------------------------------------------------------------- + + SELECT + defn + INTO + _defn + FROM + tps.srce + WHERE + srce = _srce; + + IF _defn IS NULL THEN + _message:= + format( + $$ + { + "status":"fail", + "message":"source %L does not exists" + } + $$, + _srce + )::jsonb; + RETURN _message; + END IF; + + -------------unwrap the json record and apply the path(s) of the constraint to build a constraint key per record----------------------------------------------------------------------------------- + + WITH + pending_list AS ( + SELECT + _srce srce + ,j.rec + ,j.id + --aggregate back to the record since multiple paths may be listed in the constraint + --it is unclear why the "->>0" is required to correctly extract the text array from the jsonb + ,tps.jsonb_concat_obj( + jsonb_build_object( + --the new json key is the path itself + cons.path->>0 + ,j.rec#>((cons.path->>0)::text[]) + ) + ) json_key + FROM + jsonb_array_elements(_recs) WITH ORDINALITY j(rec,id) + JOIN LATERAL jsonb_array_elements(_defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE + GROUP BY + j.rec + ,j.id + ) + + -----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------ + + , pending_keys AS ( + SELECT DISTINCT + json_key + FROM + pending_list + ) + + -----------list of keys already loaded to tps----------------------------------------------------------------------------------------------------- + + , matched_keys AS ( + SELECT DISTINCT + k.json_key + FROM + pending_keys k + INNER JOIN tps.trans t ON + t.ic = k.json_key + ) + + -----------return unique keys that are not already in tps.trans----------------------------------------------------------------------------------- + + , unmatched_keys AS ( + SELECT + json_key + FROM + pending_keys + + EXCEPT + + SELECT + json_key + FROM + matched_keys + ) + + --------build log record-------------------+------------------------------------------------------------------------------------------------ + + , logged AS ( + INSERT INTO + tps.trans_log (info) + SELECT + JSONB_BUILD_OBJECT('time_stamp',CURRENT_TIMESTAMP) + ||JSONB_BUILD_OBJECT('srce',_srce) + --||JSONB_BUILD_OBJECT('path',_path) + ||JSONB_BUILD_OBJECT('not_inserted', + ( + SELECT + jsonb_agg(json_key) + FROM + matched_keys + ) + ) + ||JSONB_BUILD_OBJECT('inserted', + ( + SELECT + jsonb_agg(json_key) + FROM + unmatched_keys + ) + ) + RETURNING * + ) + + -----------insert pending rows that have key with no trans match----------------------------------------------------------------------------------- + --need to look into mapping the transactions prior to loading + + , inserted AS ( + INSERT INTO + tps.trans (srce, rec, ic, logid) + SELECT + pl.srce + ,pl.rec + ,pl.json_key + ,logged.id + FROM + pending_list pl + INNER JOIN unmatched_keys u ON + u.json_key = pl.json_key + CROSS JOIN logged + ORDER BY + pl.id ASC + ----this conflict is only if an exact duplicate rec json happens, which will be rejected + ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified + RETURNING * + ) + + SELECT + id + ,info + INTO + _log_id + ,_log_info + FROM + logged; + + --RAISE NOTICE 'import logged under id# %, info: %', _log_id, _log_info; + + _message:= + ( + $$ + { + "status":"complete" + } + $$::jsonb + )||jsonb_build_object('details',_log_info); + + RETURN _message; + +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error importing data" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; +END; +$f$ +LANGUAGE plpgsql; + + +---------------overwrite maps-------------------------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION tps.srce_map_overwrite(_srce text) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + WITH + --------------------apply regex operations to transactions----------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT count(*) FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ORDER BY + id + ) + + + --SELECT * FROM agg_to_target + + + , link_map AS ( + SELECT + a.srce + ,a.id + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a.retain_val retain_value + ,v.map + FROM + agg_to_target a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val + ) + + --SELECT * FROM link_map + + , agg_to_id AS ( + SELECT + srce + ,id + ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val + ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map + FROM + link_map + GROUP BY + srce + ,id + ) + + --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100 + + + + UPDATE + tps.trans t + SET + map = o.map, + parse = o.retain_val, + allj = t.rec||o.map||o.retain_val + FROM + agg_to_id o + WHERE + o.id = t.id; + + _message:= jsonb_build_object('status','complete'); + RETURN _message; + +EXCEPTION WHEN OTHERS THEN + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting map value" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + + RETURN _message; +END; +$f$ +language plpgsql; + +---------------------set map values from json array of json objects----------------------------------------------------- + + +DROP FUNCTION IF EXISTS tps.map_rv_set; +CREATE OR REPLACE FUNCTION tps.map_rv_set(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; +BEGIN + INSERT INTO + tps.map_rv (srce, target, retval, map, hist) + SELECT + r.source + ,r.map + ,r.ret_val + ,r.mapped + ,jsonb_build_object( + 'hist_defn',mapped + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + FROM + JSONB_ARRAY_ELEMENTS(_defn) WITH ORDINALITY ae(r,s) + JOIN LATERAL jsonb_to_record(ae.r) r(source TEXT,map TEXT, ret_val jsonb, mapped jsonb) ON TRUE + ON CONFLICT ON CONSTRAINT map_rv_pk DO UPDATE + SET + map = excluded.map + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.map + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rv.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + + -------return message-------------------------------------------------------------------------------------------------- + _message:= jsonb_build_object('status','complete'); + RETURN _message; + +EXCEPTION WHEN OTHERS THEN + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting map value" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + + RETURN _message; +END; +$f$ +LANGUAGE plpgsql; + +------------------------------test regex with only unique results-------------------------------------------------------------- + +DROP FUNCTION IF EXISTS tps.test_regex(jsonb); +CREATE FUNCTION tps.test_regex(_defn jsonb) RETURNS jsonb +LANGUAGE plpgsql +AS +$f$ +DECLARE + _rslt jsonb; +BEGIN + + WITH + + --------------------apply regex operations to transactions--------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT * FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ) + + + , agg_to_ret AS ( + SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + FROM + agg_to_target + GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ) + ,agg_to_id AS ( + SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + FROM + agg_to_ret l + ORDER BY + l.srce + ,l.target + ,l."count" desc + ) + SELECT + jsonb_agg(row_to_json(agg_to_id)::jsonb) + INTO + _rslt + FROM + agg_to_id; + + RETURN _rslt; +END; +$f$; + +------------------------------test regex with all original records-------------------------------------------------------------- + +DROP FUNCTION IF EXISTS tps.report_unmapped_recs; +CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint, + recs jsonb + +) +LANGUAGE plpgsql +AS +$f$ +BEGIN + +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + ,jsonb_agg(rec) rec +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.rec + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + ,l.rec +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; +END; +$f$; + + +--setup function to delete a single source +DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); +CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + -------------------------------do delete--------------------------------- + + DELETE FROM tps.srce WHERE srce = _defn->>'name'; + --could move this record to a "recycle bin" table for a certain period of time + --need to handle cascading record deletes + + ---------------------------set message----------------------------------- + _message:= + ( + $$ + { + "status":"complete", + "message":"source was permanently deleted" + } + $$::jsonb + ); + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error dropping the source" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql; + +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); +CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; + _list text; +BEGIN + + WITH + --retain the results of the update by srce + _set AS ( + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + ) + --full join + ,_full AS ( + SELECT + COALESCE(_srce.srce,_set.srce) srce + ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn + ,COALESCE(_set.defn,_srce.defn) defn + FROM + tps.srce _srce + FULL OUTER JOIN _set ON + _set.srce = _srce.srce + ) + --call functions from list + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + --aggregate all the messages into one message + ---- + ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. + ---- + SELECT + jsonb_agg(m) + INTO + _message + FROM + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; + + RETURN _message; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error updating sources" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/interface/import/map_trigger.sql b/database/interface/import/map_trigger.sql new file mode 100644 index 0000000..f9215e3 --- /dev/null +++ b/database/interface/import/map_trigger.sql @@ -0,0 +1,257 @@ +CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER +AS +$f$ + DECLARE + _cnt INTEGER; + + BEGIN + IF (TG_OP = 'INSERT') THEN + + + WITH + --------------------apply regex operations to transactions----------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN new_table t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT count(*) FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ORDER BY + id + ) + + + --SELECT * FROM agg_to_target + + + , link_map AS ( + SELECT + a.srce + ,a.id + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a.retain_val retain_value + ,v.map + FROM + agg_to_target a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val + ) + + --SELECT * FROM link_map + + , agg_to_id AS ( + SELECT + srce + ,id + ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val + ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map + FROM + link_map + GROUP BY + srce + ,id + ) + + --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100 + + --create a complete list of all new inserts assuming some do not have maps (left join) + ,join_all AS ( + SELECT + n.srce + ,n.id + ,n.rec + ,a.retain_val parse + ,a.map + ,n.rec||COALESCE(a.map||a.retain_val,'{}'::jsonb) allj + FROM + new_table n + LEFT OUTER JOIN agg_to_id a ON + a.id = n.id + ) + + --update trans with join_all recs + UPDATE + tps.trans t + SET + parse = a.parse + ,map = a.map + ,allj = a.allj + FROM + join_all a + WHERE + t.id = a.id; + + + END IF; + RETURN NULL; + END; +$f$ LANGUAGE plpgsql; + +CREATE TRIGGER trans_insert + AFTER INSERT ON tps.trans + REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE tps.trans_insert_map(); \ No newline at end of file diff --git a/database/interface/import/srce_import.sql b/database/interface/import/srce_import.sql new file mode 100644 index 0000000..dd7be9c --- /dev/null +++ b/database/interface/import/srce_import.sql @@ -0,0 +1,215 @@ +DROP FUNCTION IF EXISTS tps.srce_import(text, jsonb); +CREATE OR REPLACE FUNCTION tps.srce_import(_srce text, _recs jsonb) RETURNS jsonb + +/*-------------------------------------------------------- +0. test if source exists +1. create pending list +2. get unqiue pending keys +3. see which keys not already in tps.trans +4. insert pending records associated with keys that are not already in trans +5. insert summary to log table +*/--------------------------------------------------------- + +--to-do +--return infomation to a client via json or composite type + + +AS $f$ +DECLARE + _t text; + _c text; + _log_info jsonb; + _log_id text; + _cnt numeric; + _message jsonb; + --_recs jsonb; + --_srce text; + _defn jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + --_path := 'C:\users\fleet\downloads\discover-recentactivity-20171031.csv'; + --_srce := 'dcard'; + --_recs:= $$[{"Trans. Date":"1/2/2018","Post Date":"1/2/2018","Description":"GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV","Amount":4.26,"Category":"Services"},{"Trans. Date":"1/2/2018","Post Date":"1/2/2018","Description":"MICROSOFT *ONEDRIVE 800-642-7676 WA","Amount":4.26,"Category":"Services"},{"Trans. Date":"1/3/2018","Post Date":"1/3/2018","Description":"CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3","Amount":200,"Category":"Medical Services"},{"Trans. Date":"1/4/2018","Post Date":"1/4/2018","Description":"AT&T *PAYMENT 800-288-2020 TX","Amount":57.14,"Category":"Services"},{"Trans. Date":"1/4/2018","Post Date":"1/7/2018","Description":"WWW.KOHLS.COM #0873 MIDDLETOWN OH","Amount":-7.9,"Category":"Payments and Credits"},{"Trans. Date":"1/5/2018","Post Date":"1/7/2018","Description":"PIZZA HUT 007946 STOW OH","Amount":9.24,"Category":"Restaurants"},{"Trans. Date":"1/5/2018","Post Date":"1/7/2018","Description":"SUBWAY 00044289255 STOW OH","Amount":10.25,"Category":"Restaurants"},{"Trans. Date":"1/6/2018","Post Date":"1/7/2018","Description":"ACME NO. 17 STOW OH","Amount":103.98,"Category":"Supermarkets"},{"Trans. Date":"1/6/2018","Post Date":"1/7/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":1.69,"Category":"Merchandise"},{"Trans. Date":"1/6/2018","Post Date":"1/7/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":2.19,"Category":"Merchandise"},{"Trans. Date":"1/9/2018","Post Date":"1/9/2018","Description":"CIRCLE K 05416 STOW OH00947R","Amount":3.94,"Category":"Gasoline"},{"Trans. Date":"1/9/2018","Post Date":"1/9/2018","Description":"CIRCLE K 05416 STOW OH00915R","Amount":52.99,"Category":"Gasoline"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"AUTOZONE #0722 STOW OH","Amount":85.36,"Category":"Automotive"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":26.68,"Category":"Merchandise"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"EL CAMPESINO STOW OH","Amount":6.5,"Category":"Restaurants"},{"Trans. Date":"1/13/2018","Post Date":"1/13/2018","Description":"TARGET STOW OH","Amount":197.9,"Category":"Merchandise"},{"Trans. Date":"1/14/2018","Post Date":"1/14/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":13.48,"Category":"Merchandise"},{"Trans. Date":"1/15/2018","Post Date":"1/15/2018","Description":"TARGET.COM * 800-591-3869 MN","Amount":22.41,"Category":"Merchandise"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"BUFFALO WILD WINGS KENT KENT OH","Amount":63.22,"Category":"Restaurants"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"PARTA - KCG KENT OH","Amount":4,"Category":"Government Services"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"REMEMBERNHU 402-935-7733 IA","Amount":60,"Category":"Services"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"TARGET.COM * 800-591-3869 MN","Amount":44.81,"Category":"Merchandise"},{"Trans. Date":"1/16/2018","Post Date":"1/16/2018","Description":"TREE CITY COFFEE & PASTR KENT OH","Amount":17.75,"Category":"Restaurants"},{"Trans. Date":"1/17/2018","Post Date":"1/17/2018","Description":"BESTBUYCOM805526794885 888-BESTBUY MN","Amount":343.72,"Category":"Merchandise"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"DISCOUNT DRUG MART 32 STOW OH","Amount":5.98,"Category":"Merchandise"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"U-HAUL OF KENT-STOW KENT OH","Amount":15.88,"Category":"Travel/ Entertainment"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"WALMART GROCERY 800-966-6546 AR","Amount":5.99,"Category":"Supermarkets"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"WALMART GROCERY 800-966-6546 AR","Amount":17.16,"Category":"Supermarkets"},{"Trans. Date":"1/19/2018","Post Date":"1/19/2018","Description":"WALMART GROCERY 800-966-6546 AR","Amount":500.97,"Category":"Supermarkets"},{"Trans. Date":"1/20/2018","Post Date":"1/20/2018","Description":"GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W","Amount":2.12,"Category":"Services"},{"Trans. Date":"1/20/2018","Post Date":"1/20/2018","Description":"LOWE'S OF STOW, OH. STOW OH","Amount":256.48,"Category":"Home Improvement"},{"Trans. Date":"1/23/2018","Post Date":"1/23/2018","Description":"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT","Amount":-32.2,"Category":"Awards and Rebate Credits"},{"Trans. Date":"1/23/2018","Post Date":"1/23/2018","Description":"INTERNET PAYMENT - THANK YOU","Amount":-2394.51,"Category":"Payments and Credits"},{"Trans. Date":"1/27/2018","Post Date":"1/27/2018","Description":"GIANT-EAGLE #4096 STOW OH","Amount":67.81,"Category":"Supermarkets"},{"Trans. Date":"1/27/2018","Post Date":"1/27/2018","Description":"OFFICEMAX/OFFICE DEPOT63 STOW OH","Amount":21.06,"Category":"Merchandise"},{"Trans. Date":"1/27/2018","Post Date":"1/27/2018","Description":"TARGET STOW OH","Amount":71,"Category":"Merchandise"},{"Trans. Date":"1/29/2018","Post Date":"1/29/2018","Description":"NETFLIX.COM NETFLIX.COM CA19899514437","Amount":14.93,"Category":"Services"},{"Trans. Date":"1/30/2018","Post Date":"1/30/2018","Description":"SQ *TWISTED MELTZ KENT OH0002305843011416898511","Amount":16.87,"Category":"Restaurants"},{"Trans. Date":"1/30/2018","Post Date":"1/30/2018","Description":"TARGET STOW OH","Amount":49.37,"Category":"Merchandise"}]$$::jsonb; + +----------------------------------------------------test if source exists---------------------------------------------------------------------------------- + + SELECT + defn + INTO + _defn + FROM + tps.srce + WHERE + srce = _srce; + + IF _defn IS NULL THEN + _message:= + format( + $$ + { + "status":"fail", + "message":"source %L does not exists" + } + $$, + _srce + )::jsonb; + RETURN _message; + END IF; + + -------------unwrap the json record and apply the path(s) of the constraint to build a constraint key per record----------------------------------------------------------------------------------- + + WITH + pending_list AS ( + SELECT + _srce srce + ,j.rec + ,j.id + --aggregate back to the record since multiple paths may be listed in the constraint + --it is unclear why the "->>0" is required to correctly extract the text array from the jsonb + ,tps.jsonb_concat_obj( + jsonb_build_object( + --the new json key is the path itself + cons.path->>0 + ,j.rec#>((cons.path->>0)::text[]) + ) + ) json_key + FROM + jsonb_array_elements(_recs) WITH ORDINALITY j(rec,id) + JOIN LATERAL jsonb_array_elements(_defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE + GROUP BY + j.rec + ,j.id + ) + + -----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------ + + , pending_keys AS ( + SELECT DISTINCT + json_key + FROM + pending_list + ) + + -----------list of keys already loaded to tps----------------------------------------------------------------------------------------------------- + + , matched_keys AS ( + SELECT DISTINCT + k.json_key + FROM + pending_keys k + INNER JOIN tps.trans t ON + t.ic = k.json_key + ) + + -----------return unique keys that are not already in tps.trans----------------------------------------------------------------------------------- + + , unmatched_keys AS ( + SELECT + json_key + FROM + pending_keys + + EXCEPT + + SELECT + json_key + FROM + matched_keys + ) + + --------build log record-------------------+------------------------------------------------------------------------------------------------ + + , logged AS ( + INSERT INTO + tps.trans_log (info) + SELECT + JSONB_BUILD_OBJECT('time_stamp',CURRENT_TIMESTAMP) + ||JSONB_BUILD_OBJECT('srce',_srce) + --||JSONB_BUILD_OBJECT('path',_path) + ||JSONB_BUILD_OBJECT('not_inserted', + ( + SELECT + jsonb_agg(json_key) + FROM + matched_keys + ) + ) + ||JSONB_BUILD_OBJECT('inserted', + ( + SELECT + jsonb_agg(json_key) + FROM + unmatched_keys + ) + ) + RETURNING * + ) + + -----------insert pending rows that have key with no trans match----------------------------------------------------------------------------------- + --need to look into mapping the transactions prior to loading + + , inserted AS ( + INSERT INTO + tps.trans (srce, rec, ic, logid) + SELECT + pl.srce + ,pl.rec + ,pl.json_key + ,logged.id + FROM + pending_list pl + INNER JOIN unmatched_keys u ON + u.json_key = pl.json_key + CROSS JOIN logged + ORDER BY + pl.id ASC + ----this conflict is only if an exact duplicate rec json happens, which will be rejected + ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified + RETURNING * + ) + + SELECT + id + ,info + INTO + _log_id + ,_log_info + FROM + logged; + + --RAISE NOTICE 'import logged under id# %, info: %', _log_id, _log_info; + + _message:= + ( + $$ + { + "status":"complete" + } + $$::jsonb + )||jsonb_build_object('details',_log_info); + + RETURN _message; + +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error importing data" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; +END; +$f$ +LANGUAGE plpgsql + diff --git a/database/interface/import/srce_import_do.sql b/database/interface/import/srce_import_do.sql new file mode 100644 index 0000000..a6be710 --- /dev/null +++ b/database/interface/import/srce_import_do.sql @@ -0,0 +1,184 @@ +DO +$f$ +DECLARE + _t text; + _c text; + _log_info jsonb; + _log_id text; + _cnt numeric; + _message jsonb; + _recs jsonb; + _srce text; + _defn jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + _srce := 'DMAPI'; + _recs:= $${"id":1,"doc":{"rows":[{"elements":[{"status":"OK","distance":{"text":"225 mi","value":361940},"duration":{"text":"3 hours 50 mins","value":13812}}]}],"status":"OK","origin_addresses":["Washington, DC, USA"],"destination_addresses":["New York, NY, USA"]}}$$::jsonb; + +----------------------------------------------------test if source exists---------------------------------------------------------------------------------- + + SELECT + defn + INTO + _defn + FROM + tps.srce + WHERE + srce = _srce; + + IF _defn IS NULL THEN + _message:= + format( + $$ + { + "status":"fail", + "message":"source %L does not exists" + } + $$, + _srce + )::jsonb; + RAISE NOTICE '%s', _message; + END IF; + + -------------unwrap the json record and apply the path(s) of the constraint to build a constraint key per record----------------------------------------------------------------------------------- + + WITH + pending_list AS ( + SELECT + _srce srce + ,j.rec + ,j.id + --aggregate back to the record since multiple paths may be listed in the constraint + --it is unclear why the "->>0" is required to correctly extract the text array from the jsonb + ,tps.jsonb_concat_obj( + jsonb_build_object( + --the new json key is the path itself + cons.path->>0 + ,j.rec#>((cons.path->>0)::text[]) + ) + ) json_key + FROM + jsonb_array_elements(_recs) WITH ORDINALITY j(rec,id) + JOIN LATERAL jsonb_array_elements(_defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE + GROUP BY + j.rec + ,j.id + ) + + -----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------ + + , pending_keys AS ( + SELECT DISTINCT + json_key + FROM + pending_list + ) + + -----------list of keys already loaded to tps----------------------------------------------------------------------------------------------------- + + , matched_keys AS ( + SELECT DISTINCT + k.json_key + FROM + pending_keys k + INNER JOIN tps.trans t ON + t.ic = k.json_key + ) + + -----------return unique keys that are not already in tps.trans----------------------------------------------------------------------------------- + + , unmatched_keys AS ( + SELECT + json_key + FROM + pending_keys + + EXCEPT + + SELECT + json_key + FROM + matched_keys + ) + + --------build log record-------------------+------------------------------------------------------------------------------------------------ + + , logged AS ( + INSERT INTO + tps.trans_log (info) + SELECT + JSONB_BUILD_OBJECT('time_stamp',CURRENT_TIMESTAMP) + ||JSONB_BUILD_OBJECT('srce',_srce) + --||JSONB_BUILD_OBJECT('path',_path) + ||JSONB_BUILD_OBJECT('not_inserted', + ( + SELECT + jsonb_agg(json_key) + FROM + matched_keys + ) + ) + ||JSONB_BUILD_OBJECT('inserted', + ( + SELECT + jsonb_agg(json_key) + FROM + unmatched_keys + ) + ) + RETURNING * + ) + + -----------insert pending rows that have key with no trans match----------------------------------------------------------------------------------- + --need to look into mapping the transactions prior to loading + + , inserted AS ( + INSERT INTO + tps.trans (srce, rec, ic, logid) + SELECT + pl.srce + ,pl.rec + ,pl.json_key + ,logged.id + FROM + pending_list pl + INNER JOIN unmatched_keys u ON + u.json_key = pl.json_key + CROSS JOIN logged + ORDER BY + pl.id ASC + ----this conflict is only if an exact duplicate rec json happens, which will be rejected + ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified + RETURNING * + ) + + SELECT + id + ,info + INTO + _log_id + ,_log_info + FROM + logged; + + --RAISE NOTICE 'import logged under id# %, info: %', _log_id, _log_info; + + _message:= + ( + $$ + { + "status":"complete" + } + $$::jsonb + )||jsonb_build_object('details',_log_info); + + RAISE NOTICE '%s', _message; + +END; +$f$ +LANGUAGE plpgsql + diff --git a/database/interface/map_def/srce_map_def_set.sql b/database/interface/map_def/srce_map_def_set.sql new file mode 100644 index 0000000..62a7288 --- /dev/null +++ b/database/interface/map_def/srce_map_def_set.sql @@ -0,0 +1,110 @@ +CREATE OR REPLACE FUNCTION tps.srce_map_def_set(_defn jsonb) RETURNS jsonb +AS +$f$ + +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + WITH + ------------------------------------------stage rows to insert----------------------------------------------------- + stg AS ( + SELECT + --data source + ae.r->>'srce' srce + --map name + ,ae.r->>'name' target + --map definition + ,ae.r regex + --map aggregation sequence + ,(ae.r->>'sequence')::INTEGER seq + --history definition + ,jsonb_build_object( + 'hist_defn',ae.r + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb hist + --determine if the rows are new or match + ,(m.regex->>'regex' = ae.r->>'regex')::BOOLEAN rebuild + FROM + jsonb_array_elements(_defn) ae(r) + LEFT OUTER JOIN tps.map_rm m ON + m.srce = ae.r->>'srce' + AND m.target = ae.t->>'name' + ) + ---------------------------------------do the upsert------------------------------------------------------------------- + ,ins AS ( + INSERT INTO + tps.map_rm (srce, target, regex, seq, hist) + SELECT + srce + ,target + ,regex + ,seq + ,hist + FROM + stg + ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET + srce = excluded.srce + ,target = excluded.target + ,regex = excluded.regex + ,seq = excluded.seq + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.regex + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rm.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ) + ) + ---------------------------get list of sources that had maps change-------------------------------------------------------- + , to_update AS ( + SELECT DISTINCT + srce + FROM + ins + WHERE + rebuild = TRUE + ) + --------------------------call the map overwrite for each source and return all the messages into message---------------- + /*the whole source must be overwritten because if an element is no longer returned it shoudl be wiped from the data*/ + SELECT + jsonb_agg(x.message) + INTO + _message + FROM + to_update + JOIN LATERAL tps.srce_map_overwrite(to_update.srce) AS x(message) ON TRUE; + + _message:= jsonb_build_object('status','complete','message','definition has been set'); + return _message; + + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting definition" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; + + + +END; +$f$ +language plpgsql \ No newline at end of file diff --git a/database/interface/map_def/srce_map_def_set_single.sql b/database/interface/map_def/srce_map_def_set_single.sql new file mode 100644 index 0000000..c22c488 --- /dev/null +++ b/database/interface/map_def/srce_map_def_set_single.sql @@ -0,0 +1,98 @@ +CREATE OR REPLACE FUNCTION tps.srce_map_def_set_single(_defn jsonb, _rebuild BOOLEAN) RETURNS jsonb +AS +$f$ + +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + ---------test if anythign is changing-------------------------------------------------------------------------------------------- + + IF _defn->'regex' = (SELECT regex->'regex' FROM tps.map_rm WHERE srce = _defn->>'srce' and target = _defn->>'name') THEN + _message:= + ( + $$ + { + "status":"complete", + "message":"map was not different no action taken" + } + $$::jsonb + ); + RETURN _message; + END IF; + + ---------do the rebuild----------------------------------------------------------------------------------------------------------- + + INSERT INTO + tps.map_rm (srce, target, regex, seq, hist) + SELECT + --data source + _defn->>'srce' + --map name + ,_defn->>'name' + --map definition + ,_defn + --map aggregation sequence + ,(_defn->>'sequence')::INTEGER + --history definition + ,jsonb_build_object( + 'hist_defn',_defn + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET + srce = excluded.srce + ,target = excluded.target + ,regex = excluded.regex + ,seq = excluded.seq + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.regex + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rm.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + + --------------if rebuild was flag call the rebuild-------------------------------------------------------------------------------- + + IF _rebuild THEN + SELECT + x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb + INTO + _message + FROM + tps.srce_map_overwrite(_defn->>'srce') as X(message); + END IF; + + return _message; + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + + _message:= + ($$ + { + "status":"fail", + "message":"error setting definition" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; + + + +END; +$f$ +language plpgsql \ No newline at end of file diff --git a/database/interface/map_def/test_regex.sql b/database/interface/map_def/test_regex.sql new file mode 100644 index 0000000..d09b6e9 --- /dev/null +++ b/database/interface/map_def/test_regex.sql @@ -0,0 +1,222 @@ +DROP FUNCTION IF EXISTS tps.test_regex(jsonb); +CREATE FUNCTION tps.test_regex(_defn jsonb) RETURNS jsonb +LANGUAGE plpgsql +AS +$f$ +DECLARE + _rslt jsonb; +BEGIN + + WITH + + --------------------apply regex operations to transactions--------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT * FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ) + + + , agg_to_ret AS ( + SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + FROM + agg_to_target + GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ) + ,agg_to_id AS ( + SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + FROM + agg_to_ret l + ORDER BY + l.srce + ,l.target + ,l."count" desc + ) + SELECT + jsonb_agg(row_to_json(agg_to_id)::jsonb) + INTO + _rslt + FROM + agg_to_id; + + RETURN _rslt; +END; +$f$; \ No newline at end of file diff --git a/database/interface/map_def/test_regex_recs.sql b/database/interface/map_def/test_regex_recs.sql new file mode 100644 index 0000000..2844ba3 --- /dev/null +++ b/database/interface/map_def/test_regex_recs.sql @@ -0,0 +1,211 @@ +DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb); +CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb +LANGUAGE plpgsql +AS +$f$ +DECLARE + _rslt jsonb; +BEGIN + +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention +) +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + ,jsonb_agg(rec) rec +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) +SELECT + jsonb_agg(row_to_json(agg_to_ret)::jsonb) +INTO + _rslt +FROM + agg_to_ret; +RETURN _rslt; +END; +$f$; \ No newline at end of file diff --git a/database/interface/map_values/map_rv_set.sql b/database/interface/map_values/map_rv_set.sql new file mode 100644 index 0000000..a98780f --- /dev/null +++ b/database/interface/map_values/map_rv_set.sql @@ -0,0 +1,64 @@ +DROP FUNCTION IF EXISTS tps.map_rv_set; +CREATE OR REPLACE FUNCTION tps.map_rv_set(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; +BEGIN + INSERT INTO + tps.map_rv (srce, target, retval, map, hist) + SELECT + r.source + ,r.map + ,r.ret_val + ,r.mapped + ,jsonb_build_object( + 'hist_defn',mapped + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + FROM + JSONB_ARRAY_ELEMENTS(_defn) WITH ORDINALITY ae(r,s) + JOIN LATERAL jsonb_to_record(ae.r) r(source TEXT,map TEXT, ret_val jsonb, mapped jsonb) ON TRUE + ON CONFLICT ON CONSTRAINT map_rv_pk DO UPDATE + SET + map = excluded.map + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.map + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rv.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + + -------return message-------------------------------------------------------------------------------------------------- + _message:= jsonb_build_object('status','complete'); + RETURN _message; + +EXCEPTION WHEN OTHERS THEN + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting map value" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + + RETURN _message; +END; +$f$ +LANGUAGE plpgsql; \ No newline at end of file diff --git a/database/interface/map_values/report_unmapped.sql b/database/interface/map_values/report_unmapped.sql new file mode 100644 index 0000000..30110fe --- /dev/null +++ b/database/interface/map_values/report_unmapped.sql @@ -0,0 +1,249 @@ + +DROP FUNCTION IF EXISTS tps.report_unmapped; +CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint +) +LANGUAGE plpgsql +AS +$f$ +BEGIN + +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; +END; +$f$ \ No newline at end of file diff --git a/database/interface/map_values/report_unmapped_recs.sql b/database/interface/map_values/report_unmapped_recs.sql new file mode 100644 index 0000000..8ef5899 --- /dev/null +++ b/database/interface/map_values/report_unmapped_recs.sql @@ -0,0 +1,257 @@ +DROP FUNCTION IF EXISTS tps.report_unmapped_recs; +CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint, + recs jsonb + +) +LANGUAGE plpgsql +AS +$f$ +BEGIN + +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + ,jsonb_agg(rec) rec +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.rec + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + ,l.rec +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; +END; +$f$ \ No newline at end of file diff --git a/database/interface/map_values/srce_map_overwrite.sql b/database/interface/map_values/srce_map_overwrite.sql new file mode 100644 index 0000000..4706f44 --- /dev/null +++ b/database/interface/map_values/srce_map_overwrite.sql @@ -0,0 +1,261 @@ +CREATE OR REPLACE FUNCTION tps.srce_map_overwrite(_srce text) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + WITH + --------------------apply regex operations to transactions----------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT count(*) FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ORDER BY + id + ) + + + --SELECT * FROM agg_to_target + + + , link_map AS ( + SELECT + a.srce + ,a.id + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a.retain_val retain_value + ,v.map + FROM + agg_to_target a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val + ) + + --SELECT * FROM link_map + + , agg_to_id AS ( + SELECT + srce + ,id + ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val + ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map + FROM + link_map + GROUP BY + srce + ,id + ) + + --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100 + + + + UPDATE + tps.trans t + SET + map = o.map, + parse = o.retain_val, + allj = t.rec||o.map||o.retain_val + FROM + agg_to_id o + WHERE + o.id = t.id; + + _message:= jsonb_build_object('status','complete'); + RETURN _message; + +EXCEPTION WHEN OTHERS THEN + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting map value" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + + RETURN _message; +END; +$f$ +language plpgsql \ No newline at end of file diff --git a/database/interface/source_maint/srce_build_view.sql b/database/interface/source_maint/srce_build_view.sql new file mode 100644 index 0000000..344f099 --- /dev/null +++ b/database/interface/source_maint/srce_build_view.sql @@ -0,0 +1,33 @@ +DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text); +CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT +AS +$f$ +DECLARE + --_schema text; + --_srce text; + _sql text; +BEGIN + --_schema:= 'default'; + --_srce:= 'dcard'; +SELECT + 'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';' +INTO + _sql +FROM + tps.srce s + JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE + JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE + JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE +WHERE + srce = _srce + AND list.e->>'name' = _schema +GROUP BY + s.srce + ,list.e; + +RETURN _sql; +RAISE NOTICE '%',_sql; + +END +$f$ +LANGUAGE plpgsql; \ No newline at end of file diff --git a/database/interface/source_maint/srce_delete.sql b/database/interface/source_maint/srce_delete.sql new file mode 100644 index 0000000..bbdcce9 --- /dev/null +++ b/database/interface/source_maint/srce_delete.sql @@ -0,0 +1,52 @@ +--setup function to delete a single source +DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); +CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + -------------------------------do delete--------------------------------- + + DELETE FROM tps.srce WHERE srce = _defn->>'name'; + --could move this record to a "recycle bin" table for a certain period of time + --need to handle cascading record deletes + + ---------------------------set message----------------------------------- + _message:= + ( + $$ + { + "status":"complete", + "message":"source was permanently deleted" + } + $$::jsonb + ); + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error dropping the source" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql new file mode 100644 index 0000000..ee0446a --- /dev/null +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -0,0 +1,108 @@ +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); +CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; + _list text; +BEGIN + + WITH + --retain the results of the update by srce + _set AS ( + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + ) + --full join + ,_full AS ( + SELECT + COALESCE(_srce.srce,_set.srce) srce + ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn + ,COALESCE(_set.defn,_srce.defn) defn + FROM + tps.srce _srce + FULL OUTER JOIN _set ON + _set.srce = _srce.srce + ) + --call functions from list + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + --aggregate all the messages into one message + ---- + ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. + ---- + SELECT + jsonb_agg(m) + INTO + _message + FROM + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; + + RETURN _message; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error updating sources" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/interface/source_maint/srce_set.sql b/database/interface/source_maint/srce_set.sql new file mode 100644 index 0000000..488e104 --- /dev/null +++ b/database/interface/source_maint/srce_set.sql @@ -0,0 +1,144 @@ +DROP FUNCTION IF EXISTS tps.srce_set(jsonb); +CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + ---------test if anythign is changing-------------------------------------------------------------------------------------------- + + IF _defn = (SELECT defn FROM tps.srce WHERE srce = _defn->>'name') THEN + _message:= + ( + $$ + { + "status":"complete", + "message":"source was not different no action taken" + } + $$::jsonb + ); + RETURN _message; + END IF; + + ---------if the constraint definition is changing, rebuild for existing records--------------------------------------------------- + + SELECT + NOT (_defn->'constraint' = (SELECT defn->'constraint' FROM tps.srce WHERE srce = _defn->>'name')) + INTO + _rebuild; + + RAISE NOTICE '%',_rebuild::text; + + ---------do merge----------------------------------------------------------------------------------------------------------------- + + INSERT INTO + tps.srce (srce, defn, hist) + SELECT + --extract name from defintion + _defn->>'name' + --add current timestamp to defintions + ,_defn + --add definition + ,jsonb_build_object( + 'hist_defn',_defn + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + ON CONFLICT ON CONSTRAINT srce_pkey DO UPDATE + SET + defn = _defn + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',_defn + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + srce.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + --rebuild constraint key if necessary--------------------------------------------------------------------------------------- + + IF _rebuild THEN + WITH + rebuild AS ( + SELECT + j.srce + ,j.rec + ,j.id + --aggregate back to the record since multiple paths may be listed in the constraint + ,tps.jsonb_concat_obj( + jsonb_build_object( + --the new json key is the path itself + cons.path->>0 + ,j.rec#>((cons.path->>0)::text[]) + ) + ) json_key + FROM + tps.trans j + INNER JOIN tps.srce s ON + s.srce = j.srce + JOIN LATERAL jsonb_array_elements(s.defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE + WHERE + s.srce = _defn->>'name' + GROUP BY + j.rec + ,j.id + ) + UPDATE + tps.trans t + SET + ic = r.json_key + FROM + rebuild r + WHERE + t.id = r.id; + _message:= + ( + $$ + { + "status":"complete", + "message":"source set and constraint rebuilt on existing records" + } + $$::jsonb + ); + ELSE + _message:= + ( + $$ + { + "status":"complete", + "message":"source set" + } + $$::jsonb + ); + END IF; + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error importing data" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/readme.md b/database/readme.md new file mode 100644 index 0000000..4cb3cc9 --- /dev/null +++ b/database/readme.md @@ -0,0 +1,128 @@ +Generic Data Transformation Tool +======================================================= + +The goal is to: +1. house external data and prevent duplication on insert +2. facilitate regular exression operations to extract meaningful data +3. be able to reference it from outside sources (no action required) and maintain reference to original data + + +It is well suited for data from outside systems that +* requires complex transformation (parsing and mapping) +* original data is retained for reference +* don't feel like writing a map-reduce + +use cases: +* on-going bank feeds +* jumbled product lists +* storing api results + + +The data is converted to json by the importing program and inserted to the database. +Regex expressions are applied to specified json components and the results can be mapped to other values. + + +Major Interactions +------------------------ + +* Source Definitions (Maint/Inquire) +* Regex Instructions (Maint/Inquire) +* Cross Reference List (Maint/Inquire) +* Run Import (Run Job) + + + +### Interaction Details +* _Source Definitions (Maint/Inquire)_ + + * display a list of existing sources with display detials/edit options + * create new option + * underlying function is `tps.srce_set(_name text, _defn jsonb)` + + * the current definition of a source includes data based on bad presumptions: + * how to load from a csv file using `COPY` + * setup a Postgres type to reflect the associated columns (if applicable) + + +* _Regex Instructions (Maint/Inquire)_ + + * display a list of existing instruction sets with display details/edit options + * create new option + * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json + +* _Cross Reference List (Maint/Inquire)_ + + * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` + * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs + * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` + +* _Run Import_ + + * underlying function is `tps.srce_import(_path text, _srce text)` + + + +source definition +---------------------------------------------------------------------- + +* **load data** + * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` + * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function + * build constraint key `based on srce definition` + * handle violations + * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) + * build an import log + * run maps (as opposed to relying on trigger) +* **read data** + * the `schema` key contains either a text element or a text array in curly braces + * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` + * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table + * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` + * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` + * view that `uses the source definiton` to extrapolate a table? + * a materialized table is built `based on the source definition` and any addtional regex? + * add regex = alter table add column with historic updates? + * no primary key? + * every document must work out to one row + +``` +{ + "name":"dcard", + "source":"client_file", + "loading_function":"csv" + "constraint":[ + "{Trans. Date}", + "{Post Date}" + ], + "schemas":{ + "default":[ + { + "path":"{doc,origin_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,destination_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,status}", + "type":"text", + "column_name":"status" + } + { + "path":"{doc,rows,0,elements,0,distance,value}", + "type":"numeric", + "column_name":"distance" + } + { + "path":"{doc,rows,0,elements,0,duration,value}", + "type":"numeric", + "column_name":"duration" + } + ], + "version2":[] + } +} +``` \ No newline at end of file diff --git a/database/reports/all_map_return_values.sql b/database/reports/all_map_return_values.sql new file mode 100644 index 0000000..b8b50c7 --- /dev/null +++ b/database/reports/all_map_return_values.sql @@ -0,0 +1,245 @@ +/* +first get distinct target json values +then apply regex +*/ + + +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( +SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + --------------------------json key name assigned to return value----------------------------------------------------------------------- + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + --------------------------json value resulting from regular expression----------------------------------------------------------------- + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + --------------------------push regex result into json object--------------------------------------------------------------------------- + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val +FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->>'function' = 'replace' +WHERE + --t.allj IS NULL + --t.srce = 'PNCC' AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb +/* +ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +*/ +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) + +SELECT + l.srce + ,l.target + ,l.seq + ,l.map_intention + ,l.map_val + ,l."count" + ,l.retain_val + ,l.mapped_val +FROM + link_map l +ORDER BY + l.srce + ,l.target + ,l.seq + ,l."count" desc + ,l.map_val + ,l.mapped_val \ No newline at end of file diff --git a/database/reports/colateral_balalance.sql b/database/reports/colateral_balalance.sql new file mode 100644 index 0000000..cb0e80a --- /dev/null +++ b/database/reports/colateral_balalance.sql @@ -0,0 +1,19 @@ +SELECT + r.* + ,CASE "Schedule#" + WHEN '02IN Raw Material' THEN 13097563.42 + WHEN '03IN Finished Goods' THEN 35790696.52 + ELSE 0 + END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (PARTITION BY "Schedule#" ORDER BY "Schedule#" ASC, "PostDate" ASC) running_bal + ,(LEAST("CollateralBalance" - "Ineligible Amount","MaxEligible")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral + ,(("CollateralBalance" - "Ineligible Amount")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral_nl +FROM + tpsv.pncl_default r +WHERE + "Schedule#" = '01AR' + --"Schedule#" = '02IN Raw Material' + --"Schedule#" = '03IN Finished Goods' +ORDER BY + "Schedule#" asc + ,r."PostDate" asc + ,id \ No newline at end of file diff --git a/database/reports/dcard_bal.sql b/database/reports/dcard_bal.sql new file mode 100644 index 0000000..0eb89d4 --- /dev/null +++ b/database/reports/dcard_bal.sql @@ -0,0 +1,8 @@ +\timing +SELECT + r.* + ,SUM(r."Amount") OVER (ORDER BY r."Post Date" asc , r."Description") + 1061.1 + 22.40 balance +FROM + tpsv.dcard_default r +ORDER BY + r."Post Date" asc \ No newline at end of file diff --git a/database/reports/key_list.sql b/database/reports/key_list.sql new file mode 100644 index 0000000..eef06a8 --- /dev/null +++ b/database/reports/key_list.sql @@ -0,0 +1,34 @@ +\timing + +/*-------------------------------------------------- +maintain statment level triggers to update a master log of keys +* table based listing +* composite type maintenance + +potential updates sources/events +* tps.trans insert +* tps.trans re-map +--------------------------------------------------*/ + +WITH ok AS ( + SELECT + srce, + ok.k, + jsonb_typeof(allj->ok.k) typeof, + COUNT(*) + FROM + tps.trans + JOIN LATERAL jsonb_object_keys(allj) ok(k) ON TRUE + GROUP BY + srce, + ok.k, + jsonb_typeof(allj->ok.k) + ORDER BY + srce +) +SELECT + srce + ,k + ,typeof +FROM + ok \ No newline at end of file diff --git a/database/reports/list_maps.sql b/database/reports/list_maps.sql new file mode 100644 index 0000000..f422e7e --- /dev/null +++ b/database/reports/list_maps.sql @@ -0,0 +1,19 @@ +SELECT + m.srce, + m.target, + regex->>'function' regex_function, + regex->>'where' where_clause, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number +FROM + tps.map_rm m + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true +ORDER BY + m.srce, + m.target, + e.rn \ No newline at end of file diff --git a/database/reports/loan_balance.sql b/database/reports/loan_balance.sql new file mode 100644 index 0000000..8ad0496 --- /dev/null +++ b/database/reports/loan_balance.sql @@ -0,0 +1,12 @@ +\timing +SELECT + r.*, + SUM(r."Advances"+r."Adjustments"-r."Payments") OVER (PARTITION BY "Loan#" ORDER BY r."Post Date" asc, r."Reference #" asc) +FROM + tpsv.pnco_default r +WHERE + "Loan#" = '606780191' +ORDER BY + r."Loan#" + ,r."Post Date" ASC + ,r."Reference #" ASC \ No newline at end of file diff --git a/database/templates/regex.json b/database/templates/regex.json new file mode 100644 index 0000000..a40517b --- /dev/null +++ b/database/templates/regex.json @@ -0,0 +1,25 @@ +[ + { + "regex": { + "function": "extract", + "description": "pull first 20 characters from description for mapping", + "where": [ + {} + ], + "defn": [ + { + "regex": ".{1,20}", + "map": "y", + "field": "f20", + "flag": "", + "key": "{Description}", + "retain": "y" + } + ], + "name": "First 20" + }, + "sequence": 2, + "name": "First 20", + "srce": "dcard" + } +] \ No newline at end of file diff --git a/database/templates/srce.json b/database/templates/srce.json new file mode 100644 index 0000000..daaf1e4 --- /dev/null +++ b/database/templates/srce.json @@ -0,0 +1,82 @@ +{ + "name": "dcard", + "source": "client_file", + "loading_function": "csv", + "constraint": [ + "{Trans. Date}", + "{Post Date}", + "{Description}" + ], + "schemas": [ + { + "name": "default", + "columns": [ + { + "path": "{Trans. Date}", + "type": "date", + "column_name": "Trans. Date" + }, + { + "path": "{Post Date}", + "type": "date", + "column_name": "Post Date" + }, + { + "path": "{Description}", + "type": "text", + "column_name": "Description" + }, + { + "path": "{Amount}", + "type": "numeric", + "column_name": "Amount" + }, + { + "path": "{Category}", + "type": "text", + "column_name": "Category" + } + ] + }, + { + "name": "mapped", + "columns": [ + { + "path": "{Trans. Date}", + "type": "date", + "column_name": "Trans. Date" + }, + { + "path": "{Post Date}", + "type": "date", + "column_name": "Post Date" + }, + { + "path": "{Description}", + "type": "text", + "column_name": "Description" + }, + { + "path": "{Amount}", + "type": "numeric", + "column_name": "Amount" + }, + { + "path": "{Category}", + "type": "text", + "column_name": "Category" + }, + { + "path": "{party}", + "type": "text", + "column_name": "Party" + }, + { + "path": "{reason}", + "type": "text", + "column_name": "Reason" + } + ] + } + ] +} \ No newline at end of file diff --git a/database/templates/strip_commas.jsonc b/database/templates/strip_commas.jsonc new file mode 100644 index 0000000..774661a --- /dev/null +++ b/database/templates/strip_commas.jsonc @@ -0,0 +1,24 @@ +{ + "name": "Strip Amount Commas", //the name here currently also serves as the primary key in the database + "srce": "PNCC", //name of the target source + "sequence": 1 , //only for edge cases where the instructions returns two keys of the same name, this determines priority. pretty much 1. + "regex": { //instruction set + "where": [ //only apply this regex to these specified key value pairs, if none then use empty object {} + { + "example_key":"example_value" + } + ], + "function": "replace", //even though there is an array of definitions they all have to operate under the same premise(extract or replace) + "defn": [ //there is an array of instructions + { + "key": "{Amount}", //key= the path to the json key/value pair to operate on. path woudl be a better term. + "map": "n", //y or n to indicate if the returned value will be used to search a lookup table + "flag": "g", //g indicates find all values, null or empty would be the other option I guess + "field": "amount", //the key name to give the value that comes out of this instruction + "regex": ",", //the reg expression itself + "retain": "y", //flag to indicate if the returned value should be retained and included with the data + "replace": "" //this key is only evaluated if the function is defined as replace + } + ] + } +} \ No newline at end of file diff --git a/database/templates/transaction_type.jsonc b/database/templates/transaction_type.jsonc new file mode 100644 index 0000000..56907be --- /dev/null +++ b/database/templates/transaction_type.jsonc @@ -0,0 +1,45 @@ +{ + "name": "Trans Type", + "srce": "PNCC", + "regex": { + "function": "extract", + "defn": [ + { + "key": "{AccountName}", + "map": "y", + "field": "acctn", + "regex": "(.*)", + "retain": "n" + }, + { + "key": "{Transaction}", + "map": "y", + "field": "trans", + "regex": "(.*)", + "retain": "n" + }, + { + "key": "{Description}", + "map": "y", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y" + } + ], + "where": [ + {} + ] + }, + "sequence": 1 +} + +/* +target | retval | map +------------+----------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------- + Trans Type | {"ini": "01346", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Fees"} | {"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"} + Trans Type | {"ini": "CANADA TAX", "acctn": "The HC Operating Company OPERA", "trans": "Detail Debit Adjustments"} | {"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"} + Trans Type | {"ini": "ACH DEBIT SETTLEMENT", "acctn": "The HC Operating Company OPERA", "trans": "ACH Debits"} | {"sign": "-1", "ledger": "AP - ACH", "trantype": "Disbursement"} + Trans Type | {"ini": "RET DEP ITEM RTM", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"} | {"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit RTM", "trantype": "Collections"} + Trans Type | {"ini": "RET DEP ITEM STOP", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"} | {"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit STOP", "trantype": "Collections"} + Trans Type | {"ini": "CREDIT ADJUSTMENT", "acctn": "The HC Operating Company FBO P", "trans": "Detail Credit Adjustments"} | {"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"} + */ \ No newline at end of file diff --git a/database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql b/database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql new file mode 100644 index 0000000..1f5c1d4 --- /dev/null +++ b/database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql @@ -0,0 +1,30 @@ +WITH +mod AS ( +SELECT + srce + ,jsonb_pretty(defn) orig + ,(defn - 'schemas')|| + --rebuild the schemas key value from below + jsonb_build_object( + 'schemas' + --aggregate all the new key values for a single soure + ,jsonb_agg( + --combine a new key 'name' with the columns for that name + jsonb_build_object('name',k)||jsonb_build_object('columns',v) + ) + ) rebuild +FROM + tps.srce + LEFT JOIN LATERAL jsonb_each(defn->'schemas') WITH ORDINALITY je(k,v, rn) ON TRUE +GROUP BY + srce + ,defn +) +UPDATE + tps.srce s +SET + defn = rebuild +FROM + mod +WHERE + mod.srce = s.srce \ No newline at end of file diff --git a/database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql b/database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql new file mode 100644 index 0000000..344f099 --- /dev/null +++ b/database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql @@ -0,0 +1,33 @@ +DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text); +CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT +AS +$f$ +DECLARE + --_schema text; + --_srce text; + _sql text; +BEGIN + --_schema:= 'default'; + --_srce:= 'dcard'; +SELECT + 'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';' +INTO + _sql +FROM + tps.srce s + JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE + JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE + JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE +WHERE + srce = _srce + AND list.e->>'name' = _schema +GROUP BY + s.srce + ,list.e; + +RETURN _sql; +RAISE NOTICE '%',_sql; + +END +$f$ +LANGUAGE plpgsql; \ No newline at end of file diff --git a/database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql b/database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql new file mode 100644 index 0000000..bbdcce9 --- /dev/null +++ b/database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql @@ -0,0 +1,52 @@ +--setup function to delete a single source +DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); +CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + -------------------------------do delete--------------------------------- + + DELETE FROM tps.srce WHERE srce = _defn->>'name'; + --could move this record to a "recycle bin" table for a certain period of time + --need to handle cascading record deletes + + ---------------------------set message----------------------------------- + _message:= + ( + $$ + { + "status":"complete", + "message":"source was permanently deleted" + } + $$::jsonb + ); + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error dropping the source" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql new file mode 100644 index 0000000..ee0446a --- /dev/null +++ b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql @@ -0,0 +1,108 @@ +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); +CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; + _list text; +BEGIN + + WITH + --retain the results of the update by srce + _set AS ( + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + ) + --full join + ,_full AS ( + SELECT + COALESCE(_srce.srce,_set.srce) srce + ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn + ,COALESCE(_set.defn,_srce.defn) defn + FROM + tps.srce _srce + FULL OUTER JOIN _set ON + _set.srce = _srce.srce + ) + --call functions from list + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + --aggregate all the messages into one message + ---- + ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. + ---- + SELECT + jsonb_agg(m) + INTO + _message + FROM + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; + + RETURN _message; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error updating sources" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file