Merge branch 'pt'

do_map_g_option.pgsql

@@ -77,7 +77,8 @@ FROM
         m.regex->>'function' = 'replace'
 WHERE
     --t.srce = 'PNCC'
-    rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
+    --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
+    t.map IS NULL
     --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
 ORDER BY 
     t.id DESC,
@@ -200,7 +201,7 @@ GROUP BY
 
 SELECT srce, id, jsonb_pretty(retain_val), jsonb_pretty(map) FROM agg_to_id
 
-/*
+
 UPDATE
     tps.trans t
 SET
@@ -211,4 +212,3 @@ FROM
     agg_to_id o
 WHERE
     o.id = t.id;
-*/

evt_log_gl_extract.pgsql

@@ -0,0 +1,34 @@
+
+--this is a dynamic approach that dumps all keys into the json except several that are required which it extracts
+WITH
+expand_gl AS (
+SELECT
+    id,
+    ARRAY['GL',rn::text] json_path,
+    post_stmp,
+    (a.i->>'amt')::numeric amount,
+    a.i->>'account' account,
+    j.rec->'header'->>'date' tran_date,
+    j.rec->'header'->>'vendor' vendor,
+    (a.i - '{amt,account,date}'::text[])||j.rec->'header' as therest
+FROM
+    evt.log j
+    LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(j.rec->'GL') WITH ORDINALITY a(i, rn) ON TRUE
+)
+,gl_agg AS (
+SELECT 
+    id
+    , tran_date
+    , vendor
+    , SUM(amount) amt
+    , ROUND(SUM(amount) FILTER (WHERE account = 'dcard'),2) dr
+FROM    
+    expand_gl 
+GROUP BY 
+    id
+    , tran_date
+    , vendor     
+ORDER BY 
+    id asc
+)
+SELECT id, tran_date, vendor, amt, dr, sum(dr) over(ORDER BY id) FROM gl_agg

header_item_template.pgsql

@@ -0,0 +1,72 @@
+DO $f$
+
+DECLARE _j jsonb;
+DECLARE _m text;
+
+BEGIN
+
+_j := $${"header":{"vendor":"Target","date":"10/12/2017","instrument":"Discover Card","module":"hdrio","total":47.74,"location":"Stow, OH","transaction":"purchase","offset":"dcard"},"item":[{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"shampoo","reason":"hygiene"},{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"conditioner","reason":"hygiene"},{"vend item":"BUILDING SET","amt":28.74,"account":"recreation","item":"legos","reason":"toys","qty":6,"uom":"ea"},{"vend item":"OH TAX","amt":3.02,"account":"sales tax","item":"sales tax","reason":"sales tax","rate":"0.0675"}]}$$;
+
+WITH
+j AS (
+    SELECT
+        _j  jb
+)
+
+--------build a duplicating cross join table------------------
+
+    ,os AS (
+        SELECT
+            flag, 
+            sign,
+            x.offs
+        FROM
+            j
+            JOIN LATERAL
+            (
+                VALUES
+                ('ITEM',1,null),
+                ('OFFSET',-1,j.jb->'header'->>'offset')
+            ) x (flag, sign, offs) ON TRUE
+    )
+
+
+------------do the cross join against all the item elements-------------------
+
+,build AS (
+SELECT
+    array['item',rn::text]::text jpath
+    ,COALESCE(os.offs,ae.e->>'account') acct
+    ,(ae.e->>'amt')::numeric * os.sign amount
+FROM
+    j
+    LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(J.JB->'item') WITH ORDINALITY ae(e,rn) ON TRUE
+    CROSS JOIN os
+ORDER BY
+    ae.rn ASC,
+    os.flag ASC
+)
+
+-------------re-aggregate the items into a single array point called 'gl'---------------
+
+,agg AS (
+SELECT
+    jsonb_build_object('gl',jsonb_agg(row_to_json(b))) gl
+FROM
+    build b
+)
+
+------------take the new 'gl' with array key-value pair and combine it with the original---------------
+
+SELECT
+    jsonb_pretty(agg.gl||j.jb)
+INTO
+    _m
+FROM
+    agg
+    CROSS JOIN j;
+
+    RAISE NOTICE '%', _m;
+    
+END
+$f$

log_readme.md

@@ -0,0 +1,10 @@
+the journal module is fine forvbasic items, but when entering recipts, a single item has two entries but they are hard to match up. this could be solved by creating a separate schema module that has a head, item, the glbsub items for each main item.
+
+is there a way to do this such that subsequent usage can identify any component of the json with one access path?
+
+or should each push to evt.log pre-implement the down-stream transformation to avoid this?
+
+So the main json structure woudl have a header-item, but then there woudl also be a GL array of items that are assoiated with teh othe header-item lines but not under them as heirarchy items
+The gl key then woudl be a header-item combination and coudl have a debit credit off of each of those
+
+based on inital experience with manually loading receipts, may be good to setup a receipt module that automatically sets up the offset and reverses the sign, maybe a preview of the json

map_rm_template.pgsql

@@ -150,7 +150,7 @@ FROM
     $j$::jsonb
     , 2)
 ) x;
-*/
+
 DELETE FROM tps.map_rm  where target = 'Parse ACH';
 
 INSERT INTO
@@ -249,3 +249,31 @@ FROM
     $j$::jsonb
     , 2)
 ) x;
+*/
+
+UPDATE  
+    tps.map_rm
+SET 
+    regex = 
+    $j$
+    {
+        "name":"First 20",
+        "description":"pull first 20 characters from description for mapping",
+        "defn": [
+            {
+                "key": "{Memo}",
+                "field":"f20",
+                "regex": ".{1,20}",
+                "flag":"",
+                "retain":"y"
+            }
+        ],
+        "function":"extract",
+        "map":"yes",
+        "where": [
+            {
+            }
+        ]
+    }
+    $j$::jsonb
+WHERE srce = 'HUNT'

srce.pgsql

@@ -16,21 +16,13 @@ DECLARE _t text;
 DECLARE _c text;
 DECLARE _path text;
 DECLARE _srce text;
-<<<<<<< HEAD
-
-BEGIN
-
-    _path := 'C:\users\fleet\downloads\d1026.csv';
-    _srce := 'DCARD';
-=======
 DECLARE _log_info text;
 DECLARE _log_id text;
 
 BEGIN
 
-    _path := 'C:\users\ptrowbridge\downloads\llcol.csv';
+    _path := 'C:\users\fleet\downloads\discover-recentactivity-20171031.csv';
-    _srce := 'PNCL';
+    _srce := 'DCARD';
->>>>>>> wk
 	
 ----------------------------------------------------build the column list of the temp table----------------------------------------------------------------
 

ubm_schema.sql

@@ -123,6 +123,86 @@ CREATE TYPE srce_defn_schema AS (
 );
 
 
+SET search_path = evt, pg_catalog;
+
+--
+-- Name: build_hdr_item_mje_gl(jsonb); Type: FUNCTION; Schema: evt; Owner: -
+--
+
+CREATE FUNCTION build_hdr_item_mje_gl(_j jsonb) RETURNS jsonb
+    LANGUAGE plpgsql
+    AS $_$
+DECLARE _m text;
+
+BEGIN
+
+--_j := $${"header":{"vendor":"Target","date":"10/12/2017","instrument":"Discover Card","module":"hdrio","total":47.74,"location":"Stow, OH","transaction":"purchase","offset":"dcard"},"item":[{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"shampoo","reason":"hygiene"},{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"conditioner","reason":"hygiene"},{"vend item":"BUILDING SET","amt":28.74,"account":"recreation","item":"legos","reason":"toys","qty":6,"uom":"ea"},{"vend item":"OH TAX","amt":3.02,"account":"sales tax","item":"sales tax","reason":"sales tax","rate":"0.0675"}]}$$;
+
+WITH
+j AS (
+    SELECT
+        _j  jb
+)
+
+--------build a duplicating cross join table------------------
+
+    ,os AS (
+        SELECT
+            flag, 
+            sign,
+            x.offs
+        FROM
+            j
+            JOIN LATERAL
+            (
+                VALUES
+                ('ITEM',1,null),
+                ('OFFSET',-1,j.jb->'header'->>'offset')
+            ) x (flag, sign, offs) ON TRUE
+    )
+
+
+------------do the cross join against all the item elements-------------------
+
+,build AS (
+SELECT
+    array['item',rn::text]::text jpath
+    ,COALESCE(os.offs,ae.e->>'account') acct
+    ,(ae.e->>'amt')::numeric * os.sign amount
+FROM
+    j
+    LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(J.JB->'item') WITH ORDINALITY ae(e,rn) ON TRUE
+    CROSS JOIN os
+ORDER BY
+    ae.rn ASC,
+    os.flag ASC
+)
+
+-------------re-aggregate the items into a single array point called 'gl'---------------
+
+,agg AS (
+SELECT
+    jsonb_build_object('gl',jsonb_agg(row_to_json(b))) gl
+FROM
+    build b
+)
+
+------------take the new 'gl' with array key-value pair and combine it with the original---------------
+
+SELECT
+    jsonb_pretty(agg.gl||j.jb)
+INTO
+    _j
+FROM
+    agg
+    CROSS JOIN j;
+
+RETURN _j;
+    
+END
+$_$;
+
+
 SET search_path = public, pg_catalog;
 
 --
@@ -187,7 +267,8 @@ SET default_with_oids = false;
 
 CREATE TABLE log (
     id integer NOT NULL,
-    rec jsonb
+    rec jsonb,
+    post_stmp timestamp with time zone DEFAULT now()
 );