From 3caa2b3887d20d247c68fd8d8ac86840adb683cb Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 13 Jun 2018 01:46:38 -0400 Subject: [PATCH 1/6] add notes on mapping and adjust regex testing for json schema --- deploy/setup.sql | 354 ++++++++++---------- interface/import/map_trigger.sql | 6 + interface/map_def/test_regex.sql | 165 ++++----- interface/map_def/test_regex_recs.sql | 161 ++++----- interface/map_values/srce_map_overwrite.sql | 6 + 5 files changed, 360 insertions(+), 332 deletions(-) diff --git a/deploy/setup.sql b/deploy/setup.sql index 2cb3f5e..4d6fe5b 100644 --- a/deploy/setup.sql +++ b/deploy/setup.sql @@ -302,7 +302,7 @@ BEGIN RETURN _message; END; $f$ -LANGUAGE plpgsql +LANGUAGE plpgsql; -----generate sql to create select based on schema DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text); @@ -773,14 +773,20 @@ $f$ NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN new_table t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' ORDER BY @@ -1190,7 +1196,7 @@ BEGIN t.rec, m.target, m.seq, - regex->>'function' regex_function, + regex->'regex'->>'function' regex_function, e.v ->> 'field' result_key_name, e.v ->> 'key' target_json_path, e.v ->> 'flag' regex_options_flag, @@ -1209,7 +1215,7 @@ BEGIN END map_key, CASE e.v->>'map' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(mt.mt[1]) @@ -1231,7 +1237,7 @@ BEGIN END retain_key, CASE e.v->>'retain' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(trim(mt.mt[1])) @@ -1246,16 +1252,22 @@ BEGIN NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN tps.trans t ON t.srce = m.srce AND t.rec @> w.v - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + m.regex->'regex'->>'function' = 'replace' WHERE --t.allj IS NULL t.srce = _srce @@ -1510,88 +1522,89 @@ BEGIN --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( - SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val - FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 @@ -1734,88 +1747,89 @@ WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 diff --git a/interface/import/map_trigger.sql b/interface/import/map_trigger.sql index 6002dc1..f9215e3 100644 --- a/interface/import/map_trigger.sql +++ b/interface/import/map_trigger.sql @@ -74,14 +74,20 @@ $f$ NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN new_table t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' ORDER BY diff --git a/interface/map_def/test_regex.sql b/interface/map_def/test_regex.sql index 71e928d..d09b6e9 100644 --- a/interface/map_def/test_regex.sql +++ b/interface/map_def/test_regex.sql @@ -12,88 +12,89 @@ BEGIN --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( - SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val - FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 diff --git a/interface/map_def/test_regex_recs.sql b/interface/map_def/test_regex_recs.sql index 3522b0e..2844ba3 100644 --- a/interface/map_def/test_regex_recs.sql +++ b/interface/map_def/test_regex_recs.sql @@ -12,88 +12,89 @@ WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 diff --git a/interface/map_values/srce_map_overwrite.sql b/interface/map_values/srce_map_overwrite.sql index 408f8a9..4706f44 100644 --- a/interface/map_values/srce_map_overwrite.sql +++ b/interface/map_values/srce_map_overwrite.sql @@ -74,14 +74,20 @@ BEGIN NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN tps.trans t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' WHERE From bcdb728c3d40491585ea27bf0b43016c9a4d61a4 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 16 Jun 2018 01:15:24 -0400 Subject: [PATCH 2/6] update unampped listign to reflect change in json schema --- interface/map_values/report_unmapped.sql | 195 ++++++++---------- interface/map_values/report_unmapped_recs.sql | 180 ++++++++-------- 2 files changed, 171 insertions(+), 204 deletions(-) diff --git a/interface/map_values/report_unmapped.sql b/interface/map_values/report_unmapped.sql index 5c10966..30110fe 100644 --- a/interface/map_values/report_unmapped.sql +++ b/interface/map_values/report_unmapped.sql @@ -1,24 +1,3 @@ -CREATE OR REPLACE FUNCTION tps.jsonb_concat( - state jsonb, - concat jsonb) - RETURNS jsonb AS -$BODY$ -BEGIN - --RAISE notice 'state is %', state; - --RAISE notice 'concat is %', concat; - RETURN state || concat; -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; - -DROP AGGREGATE IF EXISTS tps.jsonb_concat_obj(jsonb); -CREATE AGGREGATE tps.jsonb_concat_obj(jsonb) ( - SFUNC=tps.jsonb_concat, - STYPE=jsonb, - INITCOND='{}' -); - DROP FUNCTION IF EXISTS tps.report_unmapped; CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE @@ -44,101 +23,95 @@ WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - tps.map_rm m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -WHERE + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE --t.allj IS NULL t.srce = _srce AND e.v @> '{"map":"y"}'::jsonb --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* -ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) -*/ + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 diff --git a/interface/map_values/report_unmapped_recs.sql b/interface/map_values/report_unmapped_recs.sql index cceee91..8ef5899 100644 --- a/interface/map_values/report_unmapped_recs.sql +++ b/interface/map_values/report_unmapped_recs.sql @@ -24,101 +24,95 @@ WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - tps.map_rm m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -WHERE - --t.allj IS NULL - t.srce = _srce AND - e.v @> '{"map":"y"}'::jsonb - --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' - --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* -ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) -*/ + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 From 203fabb2c69516631e7e9838194384d2694bcd8b Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 16 Jun 2018 08:30:33 -0400 Subject: [PATCH 3/6] include changes to json schema in main sql function setup --- deploy/setup.sql | 256 +++++++++++++++++++++++++++-------------------- 1 file changed, 148 insertions(+), 108 deletions(-) diff --git a/deploy/setup.sql b/deploy/setup.sql index 4d6fe5b..0493a4a 100644 --- a/deploy/setup.sql +++ b/deploy/setup.sql @@ -439,7 +439,7 @@ CREATE AGGREGATE tps.jsonb_concat_obj(jsonb) ( ); -DROP FUNCTION IF EXISTS tps.report_unmapped(text); +DROP FUNCTION IF EXISTS tps.report_unmapped; CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE ( source text, @@ -463,101 +463,95 @@ WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - tps.map_rm m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -WHERE + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE --t.allj IS NULL t.srce = _srce AND e.v @> '{"map":"y"}'::jsonb --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* -ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) -*/ + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 @@ -1733,21 +1727,33 @@ $f$; ------------------------------test regex with all original records-------------------------------------------------------------- -DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb); -CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb +DROP FUNCTION IF EXISTS tps.report_unmapped_recs; +CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint, + recs jsonb + +) LANGUAGE plpgsql AS -$f$ -DECLARE - _rslt jsonb; +$f$ BEGIN +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( - SELECT + SELECT t.srce, t.id, t.rec, @@ -1808,24 +1814,30 @@ rx AS ( ELSE NULL END retain_val - FROM + FROM --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + tps.map_rm m --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN tps.trans t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' - ORDER BY + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY t.id DESC, m.target, e.rn, @@ -1915,6 +1927,8 @@ GROUP BY ,seq ,map_intention ) + + , agg_to_ret AS ( SELECT srce @@ -1935,12 +1949,38 @@ GROUP BY ,map_val ,retain_val ) + +, link_map AS ( SELECT - jsonb_agg(row_to_json(agg_to_ret)::jsonb) -INTO - _rslt + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.rec + ,a.retain_val + ,v.map mapped_val FROM - agg_to_ret; -RETURN _rslt; + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + ,l.rec +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; END; -$f$; \ No newline at end of file +$f$ \ No newline at end of file From d291c527492d5f3023d975933828e0b1558f7de7 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 23 Jun 2018 14:04:34 -0400 Subject: [PATCH 4/6] add sample file and adjust curl statements --- deploy/reload/dcard/curl_dcard.cmd | 8 +- deploy/reload/dcard/d.csv | 139 +++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 deploy/reload/dcard/d.csv diff --git a/deploy/reload/dcard/curl_dcard.cmd b/deploy/reload/dcard/curl_dcard.cmd index 3e2d43c..727f619 100644 --- a/deploy/reload/dcard/curl_dcard.cmd +++ b/deploy/reload/dcard/curl_dcard.cmd @@ -1,4 +1,4 @@ -curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set -curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set -curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set -curl -v -F upload=@//mnt/c/Users/fleet/Downloads/dcard.csv http://localhost:81/import?srce=dcard \ No newline at end of file +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/srce_set +curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost/mapdef_set +curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost/mapval_set +curl -v -F upload=@./d.csv http://localhost/import?srce=dcard \ No newline at end of file diff --git a/deploy/reload/dcard/d.csv b/deploy/reload/dcard/d.csv new file mode 100644 index 0000000..f0151f9 --- /dev/null +++ b/deploy/reload/dcard/d.csv @@ -0,0 +1,139 @@ +Trans. Date,Post Date,Description,Amount,Category +01/02/2018,01/02/2018,"GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV",4.26,"Services" +01/02/2018,01/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services" +01/03/2018,01/03/2018,"CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3",200.00,"Medical Services" +01/04/2018,01/04/2018,"AT&T *PAYMENT 800-288-2020 TX",57.14,"Services" +01/04/2018,01/07/2018,"WWW.KOHLS.COM #0873 MIDDLETOWN OH",-7.90,"Payments and Credits" +01/05/2018,01/07/2018,"PIZZA HUT 007946 STOW OH",9.24,"Restaurants" +01/05/2018,01/07/2018,"SUBWAY 00044289255 STOW OH",10.25,"Restaurants" +01/06/2018,01/07/2018,"ACME NO. 17 STOW OH",103.98,"Supermarkets" +01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",1.69,"Merchandise" +01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",2.19,"Merchandise" +01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00947R",3.94,"Gasoline" +01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00915R",52.99,"Gasoline" +01/13/2018,01/13/2018,"AUTOZONE #0722 STOW OH",85.36,"Automotive" +01/13/2018,01/13/2018,"DISCOUNT DRUG MART 32 STOW OH",26.68,"Merchandise" +01/13/2018,01/13/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +01/13/2018,01/13/2018,"TARGET STOW OH",197.90,"Merchandise" +01/14/2018,01/14/2018,"DISCOUNT DRUG MART 32 STOW OH",13.48,"Merchandise" +01/15/2018,01/15/2018,"TARGET.COM * 800-591-3869 MN",22.41,"Merchandise" +01/16/2018,01/16/2018,"BUFFALO WILD WINGS KENT KENT OH",63.22,"Restaurants" +01/16/2018,01/16/2018,"PARTA - KCG KENT OH",4.00,"Government Services" +01/16/2018,01/16/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services" +01/16/2018,01/16/2018,"TARGET.COM * 800-591-3869 MN",44.81,"Merchandise" +01/16/2018,01/16/2018,"TREE CITY COFFEE & PASTR KENT OH",17.75,"Restaurants" +01/17/2018,01/17/2018,"BESTBUYCOM805526794885 888-BESTBUY MN",343.72,"Merchandise" +01/19/2018,01/19/2018,"DISCOUNT DRUG MART 32 STOW OH",5.98,"Merchandise" +01/19/2018,01/19/2018,"U-HAUL OF KENT-STOW KENT OH",15.88,"Travel/ Entertainment" +01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",5.99,"Supermarkets" +01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",17.16,"Supermarkets" +01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",500.97,"Supermarkets" +01/20/2018,01/20/2018,"GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W",2.12,"Services" +01/20/2018,01/20/2018,"LOWE'S OF STOW, OH. STOW OH",256.48,"Home Improvement" +01/22/2018,01/22/2018,"HOBBY LOBBY #405 STOW OHITEM TRANSFERRED FROM PREV ACCOUNT",38.49,"Merchandise" +01/23/2018,01/23/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-32.20,"Awards and Rebate Credits" +01/23/2018,01/23/2018,"INTERNET PAYMENT - THANK YOU",-2394.51,"Payments and Credits" +01/27/2018,01/27/2018,"GIANT-EAGLE #4096 STOW OH",67.81,"Supermarkets" +01/27/2018,01/27/2018,"OFFICEMAX/OFFICE DEPOT63 STOW OH",21.06,"Merchandise" +01/27/2018,01/27/2018,"TARGET STOW OH",71.00,"Merchandise" +01/29/2018,01/29/2018,"NETFLIX.COM NETFLIX.COM CA19899514437",14.93,"Services" +01/30/2018,01/30/2018,"PARTA - KCG KENT OH",1.00,"Government Services" +01/30/2018,01/30/2018,"SPEEDWAY 09303 KEN KENT OH",46.57,"Gasoline" +01/30/2018,01/30/2018,"SQ *TWISTED MELTZ KENT OH0002305843011416898511",16.87,"Restaurants" +01/30/2018,01/30/2018,"TARGET STOW OH",49.37,"Merchandise" +01/31/2018,01/31/2018,"TARGET STOW OH",4.14,"Merchandise" +01/31/2018,01/31/2018,"TARGET STREETSBORO OH",14.28,"Merchandise" +01/31/2018,02/01/2018,"TARGET STOW OH",-21.34,"Payments and Credits" +01/31/2018,02/01/2018,"TARGET STREETSBORO OH",-9.60,"Payments and Credits" +02/01/2018,02/01/2018,"EL CAMPESINO STOW OH",42.24,"Restaurants" +02/02/2018,02/02/2018,"CASH ADVANCE FEE",5.00,"Fees" +02/02/2018,02/02/2018,"GOOGLE *ASCIIFLOW.COM G.CO/HELPPAY#CAP0HQTYN5",5.00,"Cash Advances" +02/03/2018,02/03/2018,"TARGET STREETSBORO OH",71.69,"Merchandise" +02/03/2018,02/07/2018,"SAMS CLUB - #4750 CUYAHOGA FALLOH",371.90,"Warehouse Clubs" +02/04/2018,02/04/2018,"ACME NO. 17 STOW OH",8.98,"Supermarkets" +02/04/2018,02/04/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services" +02/06/2018,02/06/2018,"MINIMUM INTEREST CHARGE FEE",0.50,"Fees" +02/06/2018,02/07/2018,"BP#954778736210 7-ELEVEN STOW OH",52.80,"Gasoline" +02/06/2018,02/07/2018,"CVS/PHARMACY #08932 TWINSBURG OH",13.87,"Merchandise" +02/07/2018,02/07/2018,"AT&T *PAYMENT 800-288-2020 TXX51Z5QX7SMT2U01",57.14,"Services" +02/07/2018,02/07/2018,"TOYS R US #9203 CUYAHOGA FALLOH",193.32,"Merchandise" +02/08/2018,02/08/2018,"GIANT-EAGLE #4096 STOW OH",66.13,"Supermarkets" +02/08/2018,02/08/2018,"TARGET STOW OH",121.32,"Merchandise" +02/09/2018,02/09/2018,"GUIDOS ORIGINAL PIZZA KENT OH",11.75,"Restaurants" +02/09/2018,02/09/2018,"MARATHON PETRO73601 TWINSBURG OH",44.30,"Gasoline" +02/10/2018,02/10/2018,"RSVP NO. 36 STOW OH",14.43,"Supermarkets" +02/10/2018,02/10/2018,"TARGET STOW OH",77.90,"Merchandise" +02/11/2018,02/11/2018,"SUBWAY 00044289255 STOW OH",21.00,"Restaurants" +02/13/2018,02/13/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",12.79,"Restaurants" +02/13/2018,02/13/2018,"IN *MR. BULKY'S FOODS AKRON OHAJ16V8Q6",3.39,"Supermarkets" +02/13/2018,02/13/2018,"TARGET CUYAHOGA FALLOH",5.33,"Supermarkets" +02/14/2018,02/14/2018,"DISCOUNT DRUG MART 32 STOW OH",4.29,"Merchandise" +02/14/2018,02/14/2018,"HANDELS ICE CREAM STOW STOW OH",7.95,"Supermarkets" +02/15/2018,02/15/2018,"BATH&BODY STOW OH",47.19,"Merchandise" +02/15/2018,02/15/2018,"TARGET STOW OH",76.35,"Merchandise" +02/17/2018,02/17/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +02/17/2018,02/17/2018,"WALMART GROCERY 800-966-6546 AR",461.36,"Supermarkets" +02/18/2018,02/18/2018,"ACME NO. 17 STOW OH",32.68,"Supermarkets" +02/18/2018,02/18/2018,"CHIPOTLE ONLINE 303-595-4000 CO",20.75,"Restaurants" +02/19/2018,02/19/2018,"GIANT EAGLE #5863 STREETSBORO OH",25.00,"Supermarkets" +02/20/2018,02/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services" +02/21/2018,02/21/2018,"BP#954635936241 7-ELEVEN STOW OH",30.04,"Gasoline" +02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",3.19,"Restaurants" +02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",18.22,"Restaurants" +02/22/2018,02/22/2018,"PET SUPPLIES PLUS #68 STOW OH",45.88,"Merchandise" +02/22/2018,02/22/2018,"TOYS R US #9203 CUYAHOGA FALLOH",21.31,"Merchandise" +02/23/2018,02/23/2018,"SUMMIT CO PARKING GAR AKRON OH",6.00,"Services" +02/24/2018,02/24/2018,"GET GO #3396 STOW OH",26.46,"Gasoline" +02/25/2018,02/25/2018,"DISCOUNT DRUG MART 32 STOW OH",19.70,"Merchandise" +02/25/2018,02/25/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +02/25/2018,02/25/2018,"SQ *CORNER CUP COFFEEH STOW OH0001152921507942036274",2.30,"Supermarkets" +02/25/2018,02/25/2018,"TARGET STOW OH",18.49,"Merchandise" +02/28/2018,02/28/2018,"NETFLIX.COM NETFLIX.COM CA20475539512",14.93,"Services" +03/01/2018,03/01/2018,"GIANT-EAGLE #4032 STOW OH",2.99,"Supermarkets" +03/01/2018,03/01/2018,"TARGET STOW OH",72.46,"Merchandise" +03/02/2018,03/02/2018,"LATE FEE",27.00,"Fees" +03/02/2018,03/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services" +03/02/2018,03/02/2018,"PIZZA HUT 007946 STOW OH",9.89,"Restaurants" +03/03/2018,03/03/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-23.28,"Awards and Rebate Credits" +03/03/2018,03/03/2018,"INTERNET PAYMENT - THANK YOU",-2451.43,"Payments and Credits" +03/03/2018,03/03/2018,"LOWE'S OF STOW, OH. STOW OH",6.93,"Home Improvement" +03/03/2018,03/07/2018,"WAL-MART SC - #2323 STOW OH",150.32,"Merchandise" +03/04/2018,03/04/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",13.66,"Merchandise" +03/06/2018,03/07/2018,"MFW BOOKS LLC 5732022000 MO",86.90,"Education" +03/06/2018,03/07/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",127.46,"Merchandise" +03/08/2018,03/08/2018,"ACME NO. 17 STOW OH",8.58,"Supermarkets" +03/08/2018,03/08/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",2.12,"Restaurants" +03/08/2018,03/08/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +03/08/2018,03/08/2018,"SPEEDWAY 03686 496 STOW OH",45.24,"Gasoline" +03/08/2018,03/08/2018,"SWENSONS STOW KENT STOW OH",8.30,"Restaurants" +03/08/2018,03/08/2018,"TOYS R US #9203 CUYAHOGA FALLOH",5.33,"Merchandise" +03/09/2018,03/09/2018,"SPEEDWAY 03686 496 STOW OH",48.29,"Gasoline" +03/10/2018,03/10/2018,"WALMART GROCERY 800-966-6546 AR",522.22,"Supermarkets" +03/11/2018,03/11/2018,"AT&T *PAYMENT 800-288-2020 TXQ8F55RY7SMT2N04",57.14,"Services" +03/11/2018,03/11/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011470140810",2.30,"Supermarkets" +03/12/2018,03/12/2018,"MICROSOFT *STORE 800-642-7676 WA",1.06,"Services" +03/15/2018,03/15/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011475075512",2.30,"Supermarkets" +03/16/2018,03/16/2018,"ACME NO. 17 STOW OH",15.85,"Supermarkets" +03/16/2018,03/16/2018,"CHIPOTLE 1152 STOW OH",3.85,"Restaurants" +03/16/2018,03/16/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +03/16/2018,03/16/2018,"PIZZA HUT 007946 STOW OH",13.98,"Restaurants" +03/17/2018,03/17/2018,"CHIPOTLE ONLINE 303-595-4000 CO",15.75,"Restaurants" +03/17/2018,03/17/2018,"DISCOUNT DRUG MART 32 STOW OH",9.89,"Merchandise" +03/17/2018,03/17/2018,"MFW BOOKS LLC 5732022000 MO",66.75,"Education" +03/18/2018,03/18/2018,"ACME NO. 17 STOW OH",27.78,"Supermarkets" +03/18/2018,03/18/2018,"GIANT-EAGLE #4032 STOW OH",28.34,"Supermarkets" +03/20/2018,03/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services" +03/20/2018,03/20/2018,"SONLIGHT CURRICULUM LTD 303-730-8193 CO",762.87,"Education" +03/21/2018,03/21/2018,"BP#954635936241 7-ELEVEN STOW OH",8.87,"Gasoline" +03/21/2018,03/21/2018,"DISCOUNT DRUG MART 32 STOW OH",18.07,"Merchandise" +03/21/2018,03/21/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011484061091",2.30,"Supermarkets" +03/21/2018,03/21/2018,"TARGET STOW OH",1.95,"Merchandise" +03/21/2018,03/21/2018,"TARGET STOW OH",224.85,"Merchandise" +03/22/2018,03/22/2018,"JUSTICE #0639 STOW OH",16.01,"Merchandise" +03/22/2018,03/22/2018,"SPEEDWAY 03686 496 STOW OH",32.54,"Gasoline" +03/22/2018,03/22/2018,"SQ *TWISTED MELTZ KENT OH0002305843011486528725",6.74,"Restaurants" +03/22/2018,03/22/2018,"TARGET STOW OH",6.60,"Merchandise" +03/25/2018,03/25/2018,"ACME NO. 17 STOW OH",95.42,"Supermarkets" +03/25/2018,03/25/2018,"ASIAN-GREEK CUISINES STOW OH",70.25,"Restaurants" +03/25/2018,03/25/2018,"MARATHON PETRO73601 TWINSBURG OH",11.09,"Gasoline" +03/25/2018,03/25/2018,"SPEEDWAY 09303 KEN KENT OH",53.28,"Gasoline" From 13de00bdf26c3de48c0b25604a42ae2d32bd33c5 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Mon, 25 Jun 2018 00:01:05 -0400 Subject: [PATCH 5/6] move everything into one folder --- LICENSE | 21 -- build_maps.xlsm => database/build_maps.xlsm | Bin .../deploy}/reload/dcard/curl_dcard.cmd | 0 .../deploy}/reload/dcard/d.csv | 0 .../deploy}/reload/dcard/extract.sql | 0 .../deploy}/reload/dcard/map.json | 0 .../deploy}/reload/dcard/srce.json | 0 .../deploy}/reload/dcard/vals.json | 0 .../deploy}/reload/hunt/curl_hunt.cmd | 0 .../deploy}/reload/hunt/extract.sql | 0 .../deploy}/reload/hunt/map.json | 0 .../deploy}/reload/hunt/srce.json | 0 .../deploy}/reload/paycom/extract.sql | 0 .../deploy}/reload/paycom/load.cmd | 0 .../deploy}/reload/paycom/map.json | 0 .../deploy}/reload/paycom/srce.json | 0 .../deploy}/reload/paycom/vals.json | 0 .../deploy}/reload/pncc/extract.sql | 0 .../deploy}/reload/pncc/load.cmd | 0 .../deploy}/reload/pncc/map.json | 0 .../deploy}/reload/pncc/srce.json | 0 .../deploy}/reload/pncc/vals.json | 0 .../deploy}/reload/pncl/extract.sql | 0 .../deploy}/reload/pncl/load.cmd | 0 .../deploy}/reload/pncl/srce.json | 0 .../deploy}/reload/pnco/extract.sql | 0 .../deploy}/reload/pnco/load.cmd | 0 .../deploy}/reload/pnco/srce.json | 0 .../deploy}/reload/wmpd/curl_hunt.cmd | 0 .../deploy}/reload/wmpd/extract.sql | 0 .../deploy}/reload/wmpd/srce.json | 0 {deploy => database/deploy}/setup.sql | 0 .../interface}/import/map_trigger.sql | 0 .../interface}/import/srce_import.sql | 0 .../interface}/import/srce_import_do.sql | 0 .../interface}/map_def/srce_map_def_set.sql | 0 .../map_def/srce_map_def_set_single.sql | 0 .../interface}/map_def/test_regex.sql | 0 .../interface}/map_def/test_regex_recs.sql | 0 .../interface}/map_values/map_rv_set.sql | 0 .../interface}/map_values/report_unmapped.sql | 0 .../map_values/report_unmapped_recs.sql | 0 .../map_values/srce_map_overwrite.sql | 0 .../source_maint/srce_build_view.sql | 0 .../interface}/source_maint/srce_set.sql | 0 readme.md => database/readme.md | 254 +++++++++--------- .../reports}/all_map_return_values.sql | 0 .../reports}/colateral_balalance.sql | 0 {reports => database/reports}/dcard_bal.sql | 0 {reports => database/reports}/key_list.sql | 0 {reports => database/reports}/list_maps.sql | 0 .../reports}/loan_balance.sql | 0 {templates => database/templates}/regex.json | 0 {templates => database/templates}/srce.json | 0 package.json | 19 -- 55 files changed, 127 insertions(+), 167 deletions(-) delete mode 100644 LICENSE rename build_maps.xlsm => database/build_maps.xlsm (100%) rename {deploy => database/deploy}/reload/dcard/curl_dcard.cmd (100%) rename {deploy => database/deploy}/reload/dcard/d.csv (100%) rename {deploy => database/deploy}/reload/dcard/extract.sql (100%) rename {deploy => database/deploy}/reload/dcard/map.json (100%) rename {deploy => database/deploy}/reload/dcard/srce.json (100%) rename {deploy => database/deploy}/reload/dcard/vals.json (100%) rename {deploy => database/deploy}/reload/hunt/curl_hunt.cmd (100%) rename {deploy => database/deploy}/reload/hunt/extract.sql (100%) rename {deploy => database/deploy}/reload/hunt/map.json (100%) rename {deploy => database/deploy}/reload/hunt/srce.json (100%) rename {deploy => database/deploy}/reload/paycom/extract.sql (100%) rename {deploy => database/deploy}/reload/paycom/load.cmd (100%) rename {deploy => database/deploy}/reload/paycom/map.json (100%) rename {deploy => database/deploy}/reload/paycom/srce.json (100%) rename {deploy => database/deploy}/reload/paycom/vals.json (100%) rename {deploy => database/deploy}/reload/pncc/extract.sql (100%) rename {deploy => database/deploy}/reload/pncc/load.cmd (100%) rename {deploy => database/deploy}/reload/pncc/map.json (100%) rename {deploy => database/deploy}/reload/pncc/srce.json (100%) rename {deploy => database/deploy}/reload/pncc/vals.json (100%) rename {deploy => database/deploy}/reload/pncl/extract.sql (100%) rename {deploy => database/deploy}/reload/pncl/load.cmd (100%) rename {deploy => database/deploy}/reload/pncl/srce.json (100%) rename {deploy => database/deploy}/reload/pnco/extract.sql (100%) rename {deploy => database/deploy}/reload/pnco/load.cmd (100%) rename {deploy => database/deploy}/reload/pnco/srce.json (100%) rename {deploy => database/deploy}/reload/wmpd/curl_hunt.cmd (100%) rename {deploy => database/deploy}/reload/wmpd/extract.sql (100%) rename {deploy => database/deploy}/reload/wmpd/srce.json (100%) rename {deploy => database/deploy}/setup.sql (100%) rename {interface => database/interface}/import/map_trigger.sql (100%) rename {interface => database/interface}/import/srce_import.sql (100%) rename {interface => database/interface}/import/srce_import_do.sql (100%) rename {interface => database/interface}/map_def/srce_map_def_set.sql (100%) rename {interface => database/interface}/map_def/srce_map_def_set_single.sql (100%) rename {interface => database/interface}/map_def/test_regex.sql (100%) rename {interface => database/interface}/map_def/test_regex_recs.sql (100%) rename {interface => database/interface}/map_values/map_rv_set.sql (100%) rename {interface => database/interface}/map_values/report_unmapped.sql (100%) rename {interface => database/interface}/map_values/report_unmapped_recs.sql (100%) rename {interface => database/interface}/map_values/srce_map_overwrite.sql (100%) rename {interface => database/interface}/source_maint/srce_build_view.sql (100%) rename {interface => database/interface}/source_maint/srce_set.sql (100%) rename readme.md => database/readme.md (97%) rename {reports => database/reports}/all_map_return_values.sql (100%) rename {reports => database/reports}/colateral_balalance.sql (100%) rename {reports => database/reports}/dcard_bal.sql (100%) rename {reports => database/reports}/key_list.sql (100%) rename {reports => database/reports}/list_maps.sql (100%) rename {reports => database/reports}/loan_balance.sql (100%) rename {templates => database/templates}/regex.json (100%) rename {templates => database/templates}/srce.json (100%) delete mode 100644 package.json diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 911ea4a..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/build_maps.xlsm b/database/build_maps.xlsm similarity index 100% rename from build_maps.xlsm rename to database/build_maps.xlsm diff --git a/deploy/reload/dcard/curl_dcard.cmd b/database/deploy/reload/dcard/curl_dcard.cmd similarity index 100% rename from deploy/reload/dcard/curl_dcard.cmd rename to database/deploy/reload/dcard/curl_dcard.cmd diff --git a/deploy/reload/dcard/d.csv b/database/deploy/reload/dcard/d.csv similarity index 100% rename from deploy/reload/dcard/d.csv rename to database/deploy/reload/dcard/d.csv diff --git a/deploy/reload/dcard/extract.sql b/database/deploy/reload/dcard/extract.sql similarity index 100% rename from deploy/reload/dcard/extract.sql rename to database/deploy/reload/dcard/extract.sql diff --git a/deploy/reload/dcard/map.json b/database/deploy/reload/dcard/map.json similarity index 100% rename from deploy/reload/dcard/map.json rename to database/deploy/reload/dcard/map.json diff --git a/deploy/reload/dcard/srce.json b/database/deploy/reload/dcard/srce.json similarity index 100% rename from deploy/reload/dcard/srce.json rename to database/deploy/reload/dcard/srce.json diff --git a/deploy/reload/dcard/vals.json b/database/deploy/reload/dcard/vals.json similarity index 100% rename from deploy/reload/dcard/vals.json rename to database/deploy/reload/dcard/vals.json diff --git a/deploy/reload/hunt/curl_hunt.cmd b/database/deploy/reload/hunt/curl_hunt.cmd similarity index 100% rename from deploy/reload/hunt/curl_hunt.cmd rename to database/deploy/reload/hunt/curl_hunt.cmd diff --git a/deploy/reload/hunt/extract.sql b/database/deploy/reload/hunt/extract.sql similarity index 100% rename from deploy/reload/hunt/extract.sql rename to database/deploy/reload/hunt/extract.sql diff --git a/deploy/reload/hunt/map.json b/database/deploy/reload/hunt/map.json similarity index 100% rename from deploy/reload/hunt/map.json rename to database/deploy/reload/hunt/map.json diff --git a/deploy/reload/hunt/srce.json b/database/deploy/reload/hunt/srce.json similarity index 100% rename from deploy/reload/hunt/srce.json rename to database/deploy/reload/hunt/srce.json diff --git a/deploy/reload/paycom/extract.sql b/database/deploy/reload/paycom/extract.sql similarity index 100% rename from deploy/reload/paycom/extract.sql rename to database/deploy/reload/paycom/extract.sql diff --git a/deploy/reload/paycom/load.cmd b/database/deploy/reload/paycom/load.cmd similarity index 100% rename from deploy/reload/paycom/load.cmd rename to database/deploy/reload/paycom/load.cmd diff --git a/deploy/reload/paycom/map.json b/database/deploy/reload/paycom/map.json similarity index 100% rename from deploy/reload/paycom/map.json rename to database/deploy/reload/paycom/map.json diff --git a/deploy/reload/paycom/srce.json b/database/deploy/reload/paycom/srce.json similarity index 100% rename from deploy/reload/paycom/srce.json rename to database/deploy/reload/paycom/srce.json diff --git a/deploy/reload/paycom/vals.json b/database/deploy/reload/paycom/vals.json similarity index 100% rename from deploy/reload/paycom/vals.json rename to database/deploy/reload/paycom/vals.json diff --git a/deploy/reload/pncc/extract.sql b/database/deploy/reload/pncc/extract.sql similarity index 100% rename from deploy/reload/pncc/extract.sql rename to database/deploy/reload/pncc/extract.sql diff --git a/deploy/reload/pncc/load.cmd b/database/deploy/reload/pncc/load.cmd similarity index 100% rename from deploy/reload/pncc/load.cmd rename to database/deploy/reload/pncc/load.cmd diff --git a/deploy/reload/pncc/map.json b/database/deploy/reload/pncc/map.json similarity index 100% rename from deploy/reload/pncc/map.json rename to database/deploy/reload/pncc/map.json diff --git a/deploy/reload/pncc/srce.json b/database/deploy/reload/pncc/srce.json similarity index 100% rename from deploy/reload/pncc/srce.json rename to database/deploy/reload/pncc/srce.json diff --git a/deploy/reload/pncc/vals.json b/database/deploy/reload/pncc/vals.json similarity index 100% rename from deploy/reload/pncc/vals.json rename to database/deploy/reload/pncc/vals.json diff --git a/deploy/reload/pncl/extract.sql b/database/deploy/reload/pncl/extract.sql similarity index 100% rename from deploy/reload/pncl/extract.sql rename to database/deploy/reload/pncl/extract.sql diff --git a/deploy/reload/pncl/load.cmd b/database/deploy/reload/pncl/load.cmd similarity index 100% rename from deploy/reload/pncl/load.cmd rename to database/deploy/reload/pncl/load.cmd diff --git a/deploy/reload/pncl/srce.json b/database/deploy/reload/pncl/srce.json similarity index 100% rename from deploy/reload/pncl/srce.json rename to database/deploy/reload/pncl/srce.json diff --git a/deploy/reload/pnco/extract.sql b/database/deploy/reload/pnco/extract.sql similarity index 100% rename from deploy/reload/pnco/extract.sql rename to database/deploy/reload/pnco/extract.sql diff --git a/deploy/reload/pnco/load.cmd b/database/deploy/reload/pnco/load.cmd similarity index 100% rename from deploy/reload/pnco/load.cmd rename to database/deploy/reload/pnco/load.cmd diff --git a/deploy/reload/pnco/srce.json b/database/deploy/reload/pnco/srce.json similarity index 100% rename from deploy/reload/pnco/srce.json rename to database/deploy/reload/pnco/srce.json diff --git a/deploy/reload/wmpd/curl_hunt.cmd b/database/deploy/reload/wmpd/curl_hunt.cmd similarity index 100% rename from deploy/reload/wmpd/curl_hunt.cmd rename to database/deploy/reload/wmpd/curl_hunt.cmd diff --git a/deploy/reload/wmpd/extract.sql b/database/deploy/reload/wmpd/extract.sql similarity index 100% rename from deploy/reload/wmpd/extract.sql rename to database/deploy/reload/wmpd/extract.sql diff --git a/deploy/reload/wmpd/srce.json b/database/deploy/reload/wmpd/srce.json similarity index 100% rename from deploy/reload/wmpd/srce.json rename to database/deploy/reload/wmpd/srce.json diff --git a/deploy/setup.sql b/database/deploy/setup.sql similarity index 100% rename from deploy/setup.sql rename to database/deploy/setup.sql diff --git a/interface/import/map_trigger.sql b/database/interface/import/map_trigger.sql similarity index 100% rename from interface/import/map_trigger.sql rename to database/interface/import/map_trigger.sql diff --git a/interface/import/srce_import.sql b/database/interface/import/srce_import.sql similarity index 100% rename from interface/import/srce_import.sql rename to database/interface/import/srce_import.sql diff --git a/interface/import/srce_import_do.sql b/database/interface/import/srce_import_do.sql similarity index 100% rename from interface/import/srce_import_do.sql rename to database/interface/import/srce_import_do.sql diff --git a/interface/map_def/srce_map_def_set.sql b/database/interface/map_def/srce_map_def_set.sql similarity index 100% rename from interface/map_def/srce_map_def_set.sql rename to database/interface/map_def/srce_map_def_set.sql diff --git a/interface/map_def/srce_map_def_set_single.sql b/database/interface/map_def/srce_map_def_set_single.sql similarity index 100% rename from interface/map_def/srce_map_def_set_single.sql rename to database/interface/map_def/srce_map_def_set_single.sql diff --git a/interface/map_def/test_regex.sql b/database/interface/map_def/test_regex.sql similarity index 100% rename from interface/map_def/test_regex.sql rename to database/interface/map_def/test_regex.sql diff --git a/interface/map_def/test_regex_recs.sql b/database/interface/map_def/test_regex_recs.sql similarity index 100% rename from interface/map_def/test_regex_recs.sql rename to database/interface/map_def/test_regex_recs.sql diff --git a/interface/map_values/map_rv_set.sql b/database/interface/map_values/map_rv_set.sql similarity index 100% rename from interface/map_values/map_rv_set.sql rename to database/interface/map_values/map_rv_set.sql diff --git a/interface/map_values/report_unmapped.sql b/database/interface/map_values/report_unmapped.sql similarity index 100% rename from interface/map_values/report_unmapped.sql rename to database/interface/map_values/report_unmapped.sql diff --git a/interface/map_values/report_unmapped_recs.sql b/database/interface/map_values/report_unmapped_recs.sql similarity index 100% rename from interface/map_values/report_unmapped_recs.sql rename to database/interface/map_values/report_unmapped_recs.sql diff --git a/interface/map_values/srce_map_overwrite.sql b/database/interface/map_values/srce_map_overwrite.sql similarity index 100% rename from interface/map_values/srce_map_overwrite.sql rename to database/interface/map_values/srce_map_overwrite.sql diff --git a/interface/source_maint/srce_build_view.sql b/database/interface/source_maint/srce_build_view.sql similarity index 100% rename from interface/source_maint/srce_build_view.sql rename to database/interface/source_maint/srce_build_view.sql diff --git a/interface/source_maint/srce_set.sql b/database/interface/source_maint/srce_set.sql similarity index 100% rename from interface/source_maint/srce_set.sql rename to database/interface/source_maint/srce_set.sql diff --git a/readme.md b/database/readme.md similarity index 97% rename from readme.md rename to database/readme.md index 71e3a1f..4cb3cc9 100644 --- a/readme.md +++ b/database/readme.md @@ -1,128 +1,128 @@ -Generic Data Transformation Tool -======================================================= - -The goal is to: -1. house external data and prevent duplication on insert -2. facilitate regular exression operations to extract meaningful data -3. be able to reference it from outside sources (no action required) and maintain reference to original data - - -It is well suited for data from outside systems that -* requires complex transformation (parsing and mapping) -* original data is retained for reference -* don't feel like writing a map-reduce - -use cases: -* on-going bank feeds -* jumbled product lists -* storing api results - - -The data is converted to json by the importing program and inserted to the database. -Regex expressions are applied to specified json components and the results can be mapped to other values. - - -Major Interactions ------------------------- - -* Source Definitions (Maint/Inquire) -* Regex Instructions (Maint/Inquire) -* Cross Reference List (Maint/Inquire) -* Run Import (Run Job) - - - -### Interaction Details -* _Source Definitions (Maint/Inquire)_ - - * display a list of existing sources with display detials/edit options - * create new option - * underlying function is `tps.srce_set(_name text, _defn jsonb)` - - * the current definition of a source includes data based on bad presumptions: - * how to load from a csv file using `COPY` - * setup a Postgres type to reflect the associated columns (if applicable) - - -* _Regex Instructions (Maint/Inquire)_ - - * display a list of existing instruction sets with display details/edit options - * create new option - * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json - -* _Cross Reference List (Maint/Inquire)_ - - * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` - * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs - * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` - -* _Run Import_ - - * underlying function is `tps.srce_import(_path text, _srce text)` - - - -source definition ----------------------------------------------------------------------- - -* **load data** - * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` - * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function - * build constraint key `based on srce definition` - * handle violations - * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) - * build an import log - * run maps (as opposed to relying on trigger) -* **read data** - * the `schema` key contains either a text element or a text array in curly braces - * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` - * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table - * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` - * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` - * view that `uses the source definiton` to extrapolate a table? - * a materialized table is built `based on the source definition` and any addtional regex? - * add regex = alter table add column with historic updates? - * no primary key? - * every document must work out to one row - -``` -{ - "name":"dcard", - "source":"client_file", - "loading_function":"csv" - "constraint":[ - "{Trans. Date}", - "{Post Date}" - ], - "schemas":{ - "default":[ - { - "path":"{doc,origin_addresses,0}", - "type":"text", - "column_name":"origin_address" - }, - { - "path":"{doc,destination_addresses,0}", - "type":"text", - "column_name":"origin_address" - }, - { - "path":"{doc,status}", - "type":"text", - "column_name":"status" - } - { - "path":"{doc,rows,0,elements,0,distance,value}", - "type":"numeric", - "column_name":"distance" - } - { - "path":"{doc,rows,0,elements,0,duration,value}", - "type":"numeric", - "column_name":"duration" - } - ], - "version2":[] - } -} +Generic Data Transformation Tool +======================================================= + +The goal is to: +1. house external data and prevent duplication on insert +2. facilitate regular exression operations to extract meaningful data +3. be able to reference it from outside sources (no action required) and maintain reference to original data + + +It is well suited for data from outside systems that +* requires complex transformation (parsing and mapping) +* original data is retained for reference +* don't feel like writing a map-reduce + +use cases: +* on-going bank feeds +* jumbled product lists +* storing api results + + +The data is converted to json by the importing program and inserted to the database. +Regex expressions are applied to specified json components and the results can be mapped to other values. + + +Major Interactions +------------------------ + +* Source Definitions (Maint/Inquire) +* Regex Instructions (Maint/Inquire) +* Cross Reference List (Maint/Inquire) +* Run Import (Run Job) + + + +### Interaction Details +* _Source Definitions (Maint/Inquire)_ + + * display a list of existing sources with display detials/edit options + * create new option + * underlying function is `tps.srce_set(_name text, _defn jsonb)` + + * the current definition of a source includes data based on bad presumptions: + * how to load from a csv file using `COPY` + * setup a Postgres type to reflect the associated columns (if applicable) + + +* _Regex Instructions (Maint/Inquire)_ + + * display a list of existing instruction sets with display details/edit options + * create new option + * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json + +* _Cross Reference List (Maint/Inquire)_ + + * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` + * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs + * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` + +* _Run Import_ + + * underlying function is `tps.srce_import(_path text, _srce text)` + + + +source definition +---------------------------------------------------------------------- + +* **load data** + * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` + * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function + * build constraint key `based on srce definition` + * handle violations + * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) + * build an import log + * run maps (as opposed to relying on trigger) +* **read data** + * the `schema` key contains either a text element or a text array in curly braces + * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` + * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table + * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` + * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` + * view that `uses the source definiton` to extrapolate a table? + * a materialized table is built `based on the source definition` and any addtional regex? + * add regex = alter table add column with historic updates? + * no primary key? + * every document must work out to one row + +``` +{ + "name":"dcard", + "source":"client_file", + "loading_function":"csv" + "constraint":[ + "{Trans. Date}", + "{Post Date}" + ], + "schemas":{ + "default":[ + { + "path":"{doc,origin_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,destination_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,status}", + "type":"text", + "column_name":"status" + } + { + "path":"{doc,rows,0,elements,0,distance,value}", + "type":"numeric", + "column_name":"distance" + } + { + "path":"{doc,rows,0,elements,0,duration,value}", + "type":"numeric", + "column_name":"duration" + } + ], + "version2":[] + } +} ``` \ No newline at end of file diff --git a/reports/all_map_return_values.sql b/database/reports/all_map_return_values.sql similarity index 100% rename from reports/all_map_return_values.sql rename to database/reports/all_map_return_values.sql diff --git a/reports/colateral_balalance.sql b/database/reports/colateral_balalance.sql similarity index 100% rename from reports/colateral_balalance.sql rename to database/reports/colateral_balalance.sql diff --git a/reports/dcard_bal.sql b/database/reports/dcard_bal.sql similarity index 100% rename from reports/dcard_bal.sql rename to database/reports/dcard_bal.sql diff --git a/reports/key_list.sql b/database/reports/key_list.sql similarity index 100% rename from reports/key_list.sql rename to database/reports/key_list.sql diff --git a/reports/list_maps.sql b/database/reports/list_maps.sql similarity index 100% rename from reports/list_maps.sql rename to database/reports/list_maps.sql diff --git a/reports/loan_balance.sql b/database/reports/loan_balance.sql similarity index 100% rename from reports/loan_balance.sql rename to database/reports/loan_balance.sql diff --git a/templates/regex.json b/database/templates/regex.json similarity index 100% rename from templates/regex.json rename to database/templates/regex.json diff --git a/templates/srce.json b/database/templates/srce.json similarity index 100% rename from templates/srce.json rename to database/templates/srce.json diff --git a/package.json b/package.json deleted file mode 100644 index 38f783d..0000000 --- a/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "tps_etl", - "version": "1.0.0", - "description": "third party source data transformation", - "main": "index.js", - "scripts": { - "test": "uh" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/fleetside72/tps_etl.git" - }, - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/fleetside72/tps_etl/issues" - }, - "homepage": "https://github.com/fleetside72/tps_etl#readme" -} From 355a1a15fa1f09a8f15952d66eeef73b9f7e6a57 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 28 Jun 2018 01:01:02 -0400 Subject: [PATCH 6/6] change to match up with api --- database/deploy/reload/dcard/curl_dcard.cmd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/database/deploy/reload/dcard/curl_dcard.cmd b/database/deploy/reload/dcard/curl_dcard.cmd index 727f619..8a77b83 100644 --- a/database/deploy/reload/dcard/curl_dcard.cmd +++ b/database/deploy/reload/dcard/curl_dcard.cmd @@ -1,4 +1,4 @@ -curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/srce_set -curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost/mapdef_set -curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost/mapval_set +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/source +curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost/regex +curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost/mapping curl -v -F upload=@./d.csv http://localhost/import?srce=dcard \ No newline at end of file