pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

drop old files and start working on json storage scripts

Browse Source
This commit is contained in:
Paul Trowbridge 2018-05-25 13:44:37 -04:00
parent c5eb8a102c
commit 0196ba4c53
15 changed files with 855 additions and 689 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
sample_discovercard/build_maps.xlsm → build_maps.xlsm
View File

184
interface/import/srce_import_do.sql Normal file
View File

@ -0,0 +1,184 @@
DO
$f$
DECLARE
_t text;
_c text;
_log_info jsonb;
_log_id text;
_cnt numeric;
_message jsonb;
_recs jsonb;
_srce text;
_defn jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
BEGIN
_srce := 'DMAPI';
_recs:= $${"id":1,"doc":{"rows":[{"elements":[{"status":"OK","distance":{"text":"225 mi","value":361940},"duration":{"text":"3 hours 50 mins","value":13812}}]}],"status":"OK","origin_addresses":["Washington, DC, USA"],"destination_addresses":["New York, NY, USA"]}}$$::jsonb;
----------------------------------------------------test if source exists----------------------------------------------------------------------------------
SELECT
defn
INTO
_defn
FROM
tps.srce
WHERE
srce = _srce;
IF _defn IS NULL THEN
_message:=
format(
$$
{
"status":"fail",
"message":"source %L does not exists"
}
$$,
_srce
)::jsonb;
RAISE NOTICE '%s', _message;
END IF;
-------------unwrap the json record and apply the path(s) of the constraint to build a constraint key per record-----------------------------------------------------------------------------------
WITH
pending_list AS (
SELECT
_srce srce
,j.rec
,j.id
--aggregate back to the record since multiple paths may be listed in the constraint
--it is unclear why the "->>0" is required to correctly extract the text array from the jsonb
,tps.jsonb_concat_obj(
jsonb_build_object(
--the new json key is the path itself
cons.path->>0
,j.rec#>((cons.path->>0)::text[])
)
) json_key
FROM
jsonb_array_elements(_recs) WITH ORDINALITY j(rec,id)
JOIN LATERAL jsonb_array_elements(_defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE
GROUP BY
j.rec
,j.id
)
-----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
, pending_keys AS (
SELECT DISTINCT
json_key
FROM
pending_list
)
-----------list of keys already loaded to tps-----------------------------------------------------------------------------------------------------
, matched_keys AS (
SELECT DISTINCT
k.json_key
FROM
pending_keys k
INNER JOIN tps.trans t ON
t.ic = k.json_key
)
-----------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
, unmatched_keys AS (
SELECT
json_key
FROM
pending_keys
EXCEPT
SELECT
json_key
FROM
matched_keys
)
--------build log record-------------------+------------------------------------------------------------------------------------------------
, logged AS (
INSERT INTO
tps.trans_log (info)
SELECT
JSONB_BUILD_OBJECT('time_stamp',CURRENT_TIMESTAMP)
||JSONB_BUILD_OBJECT('srce',_srce)
--||JSONB_BUILD_OBJECT('path',_path)
||JSONB_BUILD_OBJECT('not_inserted',
(
SELECT
jsonb_agg(json_key)
FROM
matched_keys
)
)
||JSONB_BUILD_OBJECT('inserted',
(
SELECT
jsonb_agg(json_key)
FROM
unmatched_keys
)
)
RETURNING *
)
-----------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
--need to look into mapping the transactions prior to loading
, inserted AS (
INSERT INTO
tps.trans (srce, rec, ic, logid)
SELECT
pl.srce
,pl.rec
,pl.json_key
,logged.id
FROM
pending_list pl
INNER JOIN unmatched_keys u ON
u.json_key = pl.json_key
CROSS JOIN logged
ORDER BY
pl.id ASC
----this conflict is only if an exact duplicate rec json happens, which will be rejected
----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified
RETURNING *
)
SELECT
id
,info
INTO
_log_id
,_log_info
FROM
logged;
--RAISE NOTICE 'import logged under id# %, info: %', _log_id, _log_info;
_message:=
(
$$
{
"status":"complete"
}
$$::jsonb
)||jsonb_build_object('details',_log_info);
RAISE NOTICE '%s', _message;
END;
$f$
LANGUAGE plpgsql

22
reports/col_balance.sql
View File

@ -1,22 +0,0 @@
SELECT
id
,rec->>'id'
,r.*
,CASE "Schedule#"
WHEN '02IN Raw Material' THEN 13097563.42
WHEN '03IN Finished Goods' THEN 35790696.52
ELSE 0
END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (PARTITION BY "Schedule#" ORDER BY "Schedule#" ASC, "PostDate" ASC, rec->>'id' ASC) running_bal
,(LEAST("CollateralBalance" - "Ineligible Amount","MaxEligible")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral
,(("CollateralBalance" - "Ineligible Amount")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral_nl
FROM
tps.trans
LEFT JOIN LATERAL jsonb_populate_record(null::tps.pncl, rec) r ON TRUE
WHERE
srce = 'PNCL'
--AND rec @> '{"Schedule#":"03IN Finished Goods"}'
ORDER BY
"Schedule#" asc
,r."PostDate" asc
,rec->>'id' asc

17
reports/dcard_bal.sql
View File

@ -1,17 +0,0 @@
\timing
SELECT
r."Trans. Date",
r."Post Date",
r."Description",
r."Amount",
r."Category",
rec->'id' id,
SUM(r."Amount") OVER (PARTITION BY srce ORDER BY r."Post Date" asc , rec->>'id' asc, r."Description") + 1061.1 + 22.40 balance
FROM
tps.trans
LEFT JOIN LATERAL jsonb_populate_record(null::tps.dcard, rec) r ON TRUE
WHERE
srce = 'DCARD'
ORDER BY
r."Post Date" asc
,rEC->>'id' asc

14
reports/loan_bal.sql
View File

@ -1,14 +0,0 @@
\timing
SELECT
r.*,
SUM(r."Advances"+r."Adjustments"-r."Payments") OVER (PARTITION BY "Loan#" ORDER BY r."Post Date" asc ,rec->>'id' asc, r."Reference #" asc)
FROM
tps.trans
LEFT JOIN LATERAL jsonb_populate_record(null::tps.pnco, rec) r ON TRUE
WHERE
rec @> '{"Loan#":"606780281"}'
ORDER BY
r."Loan#"
,r."Post Date" ASC
,rec->>'id' ASC
,r."Reference #" ASC

19
reports/srce_unq.sql
View File

@ -1,19 +0,0 @@
WITH
ext AS (
SELECT
srce
,defn->'unique_constraint'->>'fields'
,ARRAY(SELECT ae.e::text[] FROM jsonb_array_elements_text(defn->'unique_constraint'->'fields') ae(e)) txa
FROM
tps.srce
)
SELECT
t.srce
,jsonb_pretty(t.rec)
,jsonb_pretty(public.jsonb_extract(rec,txa))
FROM
tps.trans t
INNER JOIN ext ON
t.srce = ext.srce

18
reports/transaction_range.sql
View File

@ -1,18 +0,0 @@
\timing
SELECT
t.srce
,(ae.e::text[])[1] unq_constr
,MIN(rec #>> ae.e::text[]) min_text
,COUNT(*) cnt
,MAX(rec #>> ae.e::text[]) max_text
FROM
tps.trans t
INNER JOIN tps.srce s ON
s.srce = t.srce
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE
GROUP BY
t.srce
,(ae.e::text[])[1]
ORDER BY
t.srce
,(ae.e::text[])[1]

29
sample_google_api/data.json Normal file
View File

@ -0,0 +1,29 @@
{
"id": 1,
"doc": {
"rows": [
{
"elements": [
{
"status": "OK",
"distance": {
"text": "225 mi",
"value": 361940
},
"duration": {
"text": "3 hours 50 mins",
"value": 13812
}
}
]
}
],
"status": "OK",
"origin_addresses": [
"Washington, DC, USA"
],
"destination_addresses": [
"New York, NY, USA"
]
}
}

31
sample_google_api/def.json Normal file
View File

@ -0,0 +1,31 @@
{
"name": "DMAPI",
"type": "csv",
"schemas": {
"default": [
{
"path": "{doc,origin_addresses,0}",
"type": "text",
"column_name": "origin_address"
},
{
"path": "{doc,destination_addresses,0}",
"type": "text",
"column_name": "destination_address"
},
{
"path": "{doc,rows,0,elements,0,distance,value}",
"type": "numeric",
"column_name": "distince"
},
{
"path": "{doc,rows,0,elements,0,duration,value}",
"type": "numeric",
"column_name": "duration"
}
],
"constraint": [
"{doc}"
]
}
}

0
sample_google_api/test/import.sql Normal file
View File

38
sample_google_api/test/srce.sql Normal file
View File

@ -0,0 +1,38 @@
SELECT
jsonb_pretty(r.x)
FROM
tps.srce_set(
$$
{
"name": "DMAPI",
"type": "csv",
"schemas": {
"default": [
{
"path": "{doc,origin_addresses,0}",
"type": "text",
"column_name": "origin_address"
},
{
"path": "{doc,destination_addresses,0}",
"type": "text",
"column_name": "destination_address"
},
{
"path": "{doc,rows,0,elements,0,distance,value}",
"type": "numeric",
"column_name": "distince"
},
{
"path": "{doc,rows,0,elements,0,duration,value}",
"type": "numeric",
"column_name": "duration"
}
],
"constraint": [
"{doc}"
]
}
}
$$
) r(x);

40
sample_google_api/test_scipt.sql
View File

@ -5,24 +5,38 @@ SELECT
jsonb_pretty(r.x) jsonb_pretty(r.x)
FROM FROM
tps.srce_set( tps.srce_set(
'DMAPI',
$$ $$
{ {
"name": "DMAPI", "name": "DMAPI",
"type": "csv", "type": "csv",
"schema": [ "schemas": {
"default": [
{ {
"key": "doc", "path": "{doc,origin_addresses,0}",
"type": "jsonb" "type": "text",
"column_name": "origin_address"
},
{
"path": "{doc,destination_addresses,0}",
"type": "text",
"column_name": "destination_address"
},
{
"path": "{doc,rows,0,elements,0,distance,value}",
"type": "numeric",
"column_name": "distince"
},
{
"path": "{doc,rows,0,elements,0,duration,value}",
"type": "numeric",
"column_name": "duration"
} }
], ],
"unique_constraint": { "constraint": [
"type": "key", "{doc}"
"fields": [ ]
"{doc}"
]
}
} }
}
$$ $$
) r(x); ) r(x);
--------------------------build a csv file--------------------- --------------------------build a csv file---------------------

12
templates/insert_constraint.json
View File

@ -1,10 +1,6 @@
{ {
"unique_constraint": { "constraint": [
"{doc,origin_addresses}": [ "{doc,origin_addresses}",
"Washington, DC, USA" "{doc,destination_addresses}"
], ]
"{doc,destination_addresses}": [
"New York, NY, USA"
]
}
} }

933
templates/map_rm_template.json
View File

@ -1,462 +1,471 @@
{ [
"strip commas": {
{ "srce": "dcard",
"name": "Strip Amount Commas", "sequence": 1,
"description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric", "defn": [
"defn": [ {
{ "key": "{Description}",
"key": "{Amount}", "map": "y",
"field": "amount", "flag": "",
"regex": ",", "field": "f20",
"replace": "", "regex": ".{1,20}",
"flag": "g", "retain": "y"
"retain": "y", }
"map": "n" ],
} "name": "First 20",
], "where": [
"function": "replace", {}
"where": [ ],
{} "function": "extract",
] "description": "pull first 20 characters from description for mapping"
}, },
"Parse ACH Credits": {
{ "srce": "pncc",
"name": "Parse ACH Credits", "sequence": 1,
"description": "parse select components of the description for ACH Credits Receieved", "name": "Strip Amount Commas",
"defn": [ "description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric",
{ "defn": [
"key": "{Description}", {
"field": "beneficiary", "key": "{Amount}",
"regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "field": "amount",
"flag": "", "regex": ",",
"retain": "y", "replace": "",
"map": "n" "flag": "g",
}, "retain": "y",
{ "map": "n"
"key": "{Description}", }
"field": "Cust ID", ],
"regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "function": "replace",
"flag": "", "where": [
"retain": "y", {}
"map": "n" ]
}, },
{ {
"key": "{Description}", "srce": "pncc",
"field": "Desc", "sequence": 1,
"regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "name": "Parse ACH Credits",
"flag": "", "description": "parse select components of the description for ACH Credits Receieved",
"retain": "y", "defn": [
"map": "n" {
}, "key": "{Description}",
{ "field": "beneficiary",
"key": "{Description}", "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "originator", "flag": "",
"regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
}, "key": "{Description}",
{ "field": "Cust ID",
"key": "{Description}", "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "Batch Discr", "flag": "",
"regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
}, "key": "{Description}",
{ "field": "Desc",
"key": "{Description}", "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "Comp ID", "flag": "",
"regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
}, "key": "{Description}",
{ "field": "originator",
"key": "{Description}", "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "Addenda", "flag": "",
"regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
}, "key": "{Description}",
{ "field": "Batch Discr",
"key": "{Description}", "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "SETT", "flag": "",
"regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
}, "key": "{Description}",
{ "field": "Comp ID",
"key": "{Description}", "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "Date", "flag": "",
"regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
}, "key": "{Description}",
{ "field": "Addenda",
"key": "{Description}", "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"field": "Time", "flag": "",
"regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "retain": "y",
"flag": "", "map": "n"
"retain": "y", },
"map": "n" {
} "key": "{Description}",
], "field": "SETT",
"function": "extract", "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"where": [ "flag": "",
{ "retain": "y",
"Transaction": "ACH Credits" "map": "n"
} },
] {
}, "key": "{Description}",
"Parse ACH Debits": "field": "Date",
{ "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"name": "Parse ACH Debits", "flag": "",
"description": "parse select components of the description for ACH Credits Receieved", "retain": "y",
"defn": [ "map": "n"
{ },
"key": "{Description}", {
"field": "originator", "key": "{Description}",
"regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "field": "Time",
"flag": "", "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y", "flag": "",
"map": "n" "retain": "y",
}, "map": "n"
{ }
"key": "{Description}", ],
"field": "Cust ID", "function": "extract",
"regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "where": [
"flag": "", {
"retain": "y", "Transaction": "ACH Credits"
"map": "n" }
}, ]
{ },
"key": "{Description}", {
"field": "Desc", "srce": "pncc",
"regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "sequence": 1,
"flag": "", "name": "Parse ACH Debits",
"retain": "y", "description": "parse select components of the description for ACH Credits Receieved",
"map": "n" "defn": [
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "originator",
"field": "beneficiary", "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "Cust ID",
"field": "Batch Discr", "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "Desc",
"field": "Comp ID", "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "beneficiary",
"field": "Addenda", "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "Batch Discr",
"field": "SETT", "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "Comp ID",
"field": "Date", "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
}, {
{ "key": "{Description}",
"key": "{Description}", "field": "Addenda",
"field": "Time", "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag": "",
"flag": "", "retain": "y",
"retain": "y", "map": "n"
"map": "n" },
} {
], "key": "{Description}",
"function": "extract", "field": "SETT",
"where": [ "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
{ "flag": "",
"Transaction": "ACH Debits" "retain": "y",
} "map": "n"
] },
}, {
"Parse Wires": "key": "{Description}",
{ "field": "Date",
"name": "Parse Wires", "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered", "flag": "",
"defn": [ "retain": "y",
{ "map": "n"
"key": "{Description}", },
"field": "dparse", {
"regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", "key": "{Description}",
"flag": "g", "field": "Time",
"retain": "y", "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"map": "n" "flag": "",
}, "retain": "y",
{ "map": "n"
"key": "{Description}", }
"field": "beneficiary_components", ],
"regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", "function": "extract",
"flag": "", "where": [
"retain": "y", {
"map": "n" "Transaction": "ACH Debits"
}, }
{ ]
"key": "{Description}", },
"field": "originator_components", {
"regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", "srce": "pncc",
"flag": "", "sequence": 1,
"retain": "y", "name": "Parse Wires",
"map": "n" "description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered",
}, "defn": [
{ {
"key": "{Description}", "key": "{Description}",
"field": "beneficiary", "field": "dparse",
"regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "g",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "originator", "field": "beneficiary_components",
"regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", "regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "OBI", "field": "originator_components",
"regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "RFB", "field": "beneficiary",
"regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "ABA", "field": "originator",
"regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "BBI", "field": "OBI",
"regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "BENEBNK", "field": "RFB",
"regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "IBK", "field": "ABA",
"regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "RATE", "field": "BBI",
"regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "RECVBNK", "field": "BENEBNK",
"regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)", "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "", "flag": "",
"retain": "y", "retain": "y",
"map": "n" "map": "n"
} },
], {
"function": "extract", "key": "{Description}",
"where": [ "field": "IBK",
{ "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)",
"Transaction": "Money Transfer DB - Wire" "flag": "",
}, "retain": "y",
{ "map": "n"
"Transaction": "Money Transfer DB - Other" },
}, {
{ "key": "{Description}",
"Transaction": "Money Transfer CR-Wire" "field": "RATE",
}, "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)",
{ "flag": "",
"Transaction": "Money Transfer CR-Other" "retain": "y",
}, "map": "n"
{ },
"Transaction": "Intl Money Transfer Debits" {
}, "key": "{Description}",
{ "field": "RECVBNK",
"Transaction": "Intl Money Transfer Credits" "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)",
} "flag": "",
] "retain": "y",
}, "map": "n"
"Trans Type": }
{ ],
"name": "Trans Type", "function": "extract",
"description": "extract intial description in conjunction with account name and transaction type for mapping", "where": [
"defn": [ {
{ "Transaction": "Money Transfer DB - Wire"
"key": "{AccountName}", },
"field": "acctn", {
"regex": "(.*)", "Transaction": "Money Transfer DB - Other"
"retain": "n", },
"map": "y" {
}, "Transaction": "Money Transfer CR-Wire"
{ },
"key": "{Transaction}", {
"field": "trans", "Transaction": "Money Transfer CR-Other"
"regex": "(.*)", },
"retain": "n", {
"map": "y" "Transaction": "Intl Money Transfer Debits"
}, },
{ {
"key": "{Description}", "Transaction": "Intl Money Transfer Credits"
"field": "ini", }
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", ]
"retain": "y", },
"map": "y" {
} "srce": "pncc",
], "sequence": 1,
"where": [ "name": "Trans Type",
{} "description": "extract intial description in conjunction with account name and transaction type for mapping",
], "defn": [
"function": "extract" {
}, "key": "{AccountName}",
"Currency": "field": "acctn",
{ "regex": "(.*)",
"name": "Currency", "retain": "n",
"description": "pull out currency indicators from description of misc items and map", "map": "y"
"defn": [ },
{ {
"key": "{Description}", "key": "{Transaction}",
"field": "ini", "field": "trans",
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", "regex": "(.*)",
"retain": "y", "retain": "n",
"map": "y" "map": "y"
}, },
{ {
"key": "{Description}", "key": "{Description}",
"field": "curr1", "field": "ini",
"regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"retain": "y", "retain": "y",
"map": "y" "map": "y"
}, }
{ ],
"key": "{Description}", "where": [
"field": "curr2", {}
"regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", ],
"retain": "y", "function": "extract"
"map": "y" },
} {
], "srce": "pncc",
"where": [ "sequence": 1,
{ "name": "Currency",
"Transaction": "Miscellaneous Credits" "description": "pull out currency indicators from description of misc items and map",
}, "defn": [
{ {
"Transaction": "Miscellaneous Debits" "key": "{Description}",
} "field": "ini",
], "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"function": "extract" "retain": "y",
}, "map": "y"
"check number": },
{ {
"defn": [ "key": "{Description}",
{ "field": "curr1",
"key": "{Description}", "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*",
"field": "checkn", "retain": "y",
"regex": "[^0-9]*([0-9]*)\\s|$", "map": "y"
"retain": "y", },
"map": "n" {
} "key": "{Description}",
], "field": "curr2",
"where": [ "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*",
{ "retain": "y",
"Transaction": "Checks Paid" "map": "y"
} }
], ],
"function": "extract" "where": [
}, {
"ADP Codes": "Transaction": "Miscellaneous Credits"
{ },
"name": "ADP Codes", {
"description": "link to adp code definitions", "Transaction": "Miscellaneous Debits"
"defn": [ }
{ ],
"key": "{gl_descr}", "function": "extract"
"field": "gl_descr", },
"regex": ".*", {
"flag": "", "srce": "adprp",
"retain": "n", "sequence": 1,
"map": "y" "name": "ADP Codes",
}, "description": "link to adp code definitions",
{ "defn": [
"key": "{prim_offset}", {
"field": "prim_offset", "key": "{gl_descr}",
"regex": ".*", "field": "gl_descr",
"flag": "", "regex": ".*",
"retain": "n", "flag": "",
"map": "y" "retain": "n",
}, "map": "y"
{ },
"key": "{pay_date}", {
"field": "pay_month", "key": "{prim_offset}",
"regex": ".{1,4}", "field": "prim_offset",
"flag": "", "regex": ".*",
"retain": "y", "flag": "",
"map": "n" "retain": "n",
} "map": "y"
], },
"function": "extract", {
"where": [ "key": "{pay_date}",
{} "field": "pay_month",
] "regex": ".{1,4}",
} "flag": "",
} "retain": "y",
"map": "n"
}
],
"function": "extract",
"where": [
{}
]
}
]

187
templates/srce_template.json
View File

@ -1,121 +1,76 @@
{ {
"name": "WMPD", "name": "dcard",
"descr": "Williams Paid File", "source": "client_file",
"type":"csv", "loading_function": "csv",
"schema": [ "constraint": [
{ "{Trans. Date}",
"key": "Carrier", "{Post Date}",
"type": "text" "{Description}"
},
{
"key": "SCAC",
"type": "text"
},
{
"key": "Mode",
"type": "text"
},
{
"key": "Pro #",
"type": "text"
},
{
"key": "B/L",
"type": "text"
},
{
"key": "Pd Amt",
"type": "numeric"
},
{
"key": "Loc#",
"type": "text"
},
{
"key": "Pcs",
"type": "numeric"
},
{
"key": "Wgt",
"type": "numeric"
},
{
"key": "Chk#",
"type": "numeric"
},
{
"key": "Pay Dt",
"type": "date"
},
{
"key": "Acct #",
"type": "text"
},
{
"key": "I/O",
"type": "text"
},
{
"key": "Sh Nm",
"type": "text"
},
{
"key": "Sh City",
"type": "text"
},
{
"key": "Sh St",
"type": "text"
},
{
"key": "Sh Zip",
"type": "text"
},
{
"key": "Cons Nm",
"type": "text"
},
{
"key": "D City ",
"type": "text"
},
{
"key": "D St",
"type": "text"
},
{
"key": "D Zip",
"type": "text"
},
{
"key": "Sh Dt",
"type": "date"
},
{
"key": "Inv Dt",
"type": "date"
},
{
"key": "Customs Entry#",
"type": "text"
},
{
"key": "Miles",
"type": "numeric"
},
{
"key": "Frt Class",
"type": "text"
},
{
"key": "Master B/L",
"type": "text"
}
], ],
"unique_constraint": { "schemas": {
"fields":[ "default": [
"{Pay Dt}", {
"{Carrier}" "path": "{Trans. Date}",
"type": "date",
"column_name": "Trans. Date"
},
{
"path": "{Post Date}",
"type": "date",
"column_name": "Post Date"
},
{
"path": "{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category}",
"type": "text",
"column_name": "Category"
}
],
"mapped": [
{
"path": "{Trans. Date}",
"type": "date",
"column_name": "Trans. Date"
},
{
"path": "{Post Date}",
"type": "date",
"column_name": "Post Date"
},
{
"path": "{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category}",
"type": "text",
"column_name": "Category"
},
{
"path": "{party}",
"type": "text",
"column_name": "Party"
},
{
"path": "{reason}",
"type": "text",
"column_name": "Reason"
}
] ]
} }
} }