pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'pt'

Browse Source
This commit is contained in:
Paul Trowbridge 2017-11-04 09:19:27 -04:00
commit f6e418c8a0
11 changed files with 2951 additions and 2700 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
build_json.xlsx
View File

Binary file not shown.

6
do_map_g_option.pgsql
View File

@ -77,7 +77,8 @@ FROM
m.regex->>'function' = 'replace'
WHERE
--t.srce = 'PNCC'
rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
--rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
t.map IS NULL
--rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
ORDER BY
t.id DESC,
@ -200,7 +201,7 @@ GROUP BY
SELECT srce, id, jsonb_pretty(retain_val), jsonb_pretty(map) FROM agg_to_id
/*
UPDATE
tps.trans t
SET
@ -211,4 +212,3 @@ FROM
agg_to_id o
WHERE
o.id = t.id;
*/

34
evt_log_gl_extract.pgsql Normal file
View File

@ -0,0 +1,34 @@
--this is a dynamic approach that dumps all keys into the json except several that are required which it extracts
WITH
expand_gl AS (
SELECT
id,
ARRAY['GL',rn::text] json_path,
post_stmp,
(a.i->>'amt')::numeric amount,
a.i->>'account' account,
j.rec->'header'->>'date' tran_date,
j.rec->'header'->>'vendor' vendor,
(a.i - '{amt,account,date}'::text[])||j.rec->'header' as therest
FROM
evt.log j
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(j.rec->'GL') WITH ORDINALITY a(i, rn) ON TRUE
)
,gl_agg AS (
SELECT
id
, tran_date
, vendor
, SUM(amount) amt
, ROUND(SUM(amount) FILTER (WHERE account = 'dcard'),2) dr
FROM
expand_gl
GROUP BY
id
, tran_date
, vendor
ORDER BY
id asc
)
SELECT id, tran_date, vendor, amt, dr, sum(dr) over(ORDER BY id) FROM gl_agg

72
header_item_template.pgsql Normal file
View File

@ -0,0 +1,72 @@
DO $f$
DECLARE _j jsonb;
DECLARE _m text;
BEGIN
_j := $${"header":{"vendor":"Target","date":"10/12/2017","instrument":"Discover Card","module":"hdrio","total":47.74,"location":"Stow, OH","transaction":"purchase","offset":"dcard"},"item":[{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"shampoo","reason":"hygiene"},{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"conditioner","reason":"hygiene"},{"vend item":"BUILDING SET","amt":28.74,"account":"recreation","item":"legos","reason":"toys","qty":6,"uom":"ea"},{"vend item":"OH TAX","amt":3.02,"account":"sales tax","item":"sales tax","reason":"sales tax","rate":"0.0675"}]}$$;
WITH
j AS (
SELECT
_j jb
)
--------build a duplicating cross join table------------------
,os AS (
SELECT
flag,
sign,
x.offs
FROM
j
JOIN LATERAL
(
VALUES
('ITEM',1,null),
('OFFSET',-1,j.jb->'header'->>'offset')
) x (flag, sign, offs) ON TRUE
)
------------do the cross join against all the item elements-------------------
,build AS (
SELECT
array['item',rn::text]::text jpath
,COALESCE(os.offs,ae.e->>'account') acct
,(ae.e->>'amt')::numeric * os.sign amount
FROM
j
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(J.JB->'item') WITH ORDINALITY ae(e,rn) ON TRUE
CROSS JOIN os
ORDER BY
ae.rn ASC,
os.flag ASC
)
-------------re-aggregate the items into a single array point called 'gl'---------------
,agg AS (
SELECT
jsonb_build_object('gl',jsonb_agg(row_to_json(b))) gl
FROM
build b
)
------------take the new 'gl' with array key-value pair and combine it with the original---------------
SELECT
jsonb_pretty(agg.gl||j.jb)
INTO
_m
FROM
agg
CROSS JOIN j;
RAISE NOTICE '%', _m;
END
$f$

10
log_readme.md Normal file
View File

@ -0,0 +1,10 @@
the journal module is fine forvbasic items, but when entering recipts, a single item has two entries but they are hard to match up. this could be solved by creating a separate schema module that has a head, item, the glbsub items for each main item.
is there a way to do this such that subsequent usage can identify any component of the json with one access path?
or should each push to evt.log pre-implement the down-stream transformation to avoid this?
So the main json structure woudl have a header-item, but then there woudl also be a GL array of items that are assoiated with teh othe header-item lines but not under them as heirarchy items
The gl key then woudl be a header-item combination and coudl have a debit credit off of each of those
based on inital experience with manually loading receipts, may be good to setup a receipt module that automatically sets up the offset and reverses the sign, maybe a preview of the json

30
map_rm_template.pgsql
View File

@ -150,7 +150,7 @@ FROM
$j$::jsonb
, 2)
) x;
*/
DELETE FROM tps.map_rm where target = 'Parse ACH';
INSERT INTO
@ -249,3 +249,31 @@ FROM
$j$::jsonb
, 2)
) x;
*/
UPDATE
tps.map_rm
SET
regex =
$j$
{
"name":"First 20",
"description":"pull first 20 characters from description for mapping",
"defn": [
{
"key": "{Memo}",
"field":"f20",
"regex": ".{1,20}",
"flag":"",
"retain":"y"
}
],
"function":"extract",
"map":"yes",
"where": [
{
}
]
}
$j$::jsonb
WHERE srce = 'HUNT'

22
new_format.pgsql
View File

File diff suppressed because one or more lines are too long

12
srce.pgsql
View File

@ -16,21 +16,13 @@ DECLARE _t text;
DECLARE _c text;
DECLARE _path text;
DECLARE _srce text;
<<<<<<< HEAD
BEGIN
_path := 'C:\users\fleet\downloads\d1026.csv';
_srce := 'DCARD';
=======
DECLARE _log_info text;
DECLARE _log_id text;
BEGIN
_path := 'C:\users\ptrowbridge\downloads\llcol.csv';
_srce := 'PNCL';
>>>>>>> wk
_path := 'C:\users\fleet\downloads\discover-recentactivity-20171031.csv';
_srce := 'DCARD';
----------------------------------------------------build the column list of the temp table----------------------------------------------------------------

BIN
summary.xlsx Normal file
View File

Binary file not shown.

5380
ubm_data.sql
View File

File diff suppressed because one or more lines are too long

83
ubm_schema.sql
View File

@ -123,6 +123,86 @@ CREATE TYPE srce_defn_schema AS (
);
SET search_path = evt, pg_catalog;
--
-- Name: build_hdr_item_mje_gl(jsonb); Type: FUNCTION; Schema: evt; Owner: -
--
CREATE FUNCTION build_hdr_item_mje_gl(_j jsonb) RETURNS jsonb
LANGUAGE plpgsql
AS $_$
DECLARE _m text;
BEGIN
--_j := $${"header":{"vendor":"Target","date":"10/12/2017","instrument":"Discover Card","module":"hdrio","total":47.74,"location":"Stow, OH","transaction":"purchase","offset":"dcard"},"item":[{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"shampoo","reason":"hygiene"},{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"conditioner","reason":"hygiene"},{"vend item":"BUILDING SET","amt":28.74,"account":"recreation","item":"legos","reason":"toys","qty":6,"uom":"ea"},{"vend item":"OH TAX","amt":3.02,"account":"sales tax","item":"sales tax","reason":"sales tax","rate":"0.0675"}]}$$;
WITH
j AS (
SELECT
_j jb
)
--------build a duplicating cross join table------------------
,os AS (
SELECT
flag,
sign,
x.offs
FROM
j
JOIN LATERAL
(
VALUES
('ITEM',1,null),
('OFFSET',-1,j.jb->'header'->>'offset')
) x (flag, sign, offs) ON TRUE
)
------------do the cross join against all the item elements-------------------
,build AS (
SELECT
array['item',rn::text]::text jpath
,COALESCE(os.offs,ae.e->>'account') acct
,(ae.e->>'amt')::numeric * os.sign amount
FROM
j
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(J.JB->'item') WITH ORDINALITY ae(e,rn) ON TRUE
CROSS JOIN os
ORDER BY
ae.rn ASC,
os.flag ASC
)
-------------re-aggregate the items into a single array point called 'gl'---------------
,agg AS (
SELECT
jsonb_build_object('gl',jsonb_agg(row_to_json(b))) gl
FROM
build b
)
------------take the new 'gl' with array key-value pair and combine it with the original---------------
SELECT
jsonb_pretty(agg.gl||j.jb)
INTO
_j
FROM
agg
CROSS JOIN j;
RETURN _j;
END
$_$;
SET search_path = public, pg_catalog;
--
@ -187,7 +267,8 @@ SET default_with_oids = false;
CREATE TABLE log (
id integer NOT NULL,
rec jsonb
rec jsonb,
post_stmp timestamp with time zone DEFAULT now()
);