pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

push inserts into do block so they can consume local variables, need to setup log inserts

Browse Source
This commit is contained in:
Paul Trowbridge 2017-10-23 10:12:37 -04:00
parent b3775210b5
commit 51fbec1244
1 changed files with 113 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
srce.pgsql
View File

@ -15,9 +15,14 @@ DO $$
DECLARE _t text;
DECLARE _c text;
DECLARE _path text;
DECLARE _srce text;
BEGIN
_path := 'C:\users\ptrowbridge\downloads\lon_loan_ledgerbal.csv';
_srce := 'PNCO';
----------------------------------------------------build the column list of the temp table----------------------------------------------------------------
SELECT
@ -31,7 +36,7 @@ BEGIN
--unwrap the schema definition array
LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE
WHERE
srce = 'PNCC'
srce = _srce
GROUP BY
srce;
@ -48,22 +53,17 @@ BEGIN
----------------------------------------------------do the insert-------------------------------------------------------------------------------------------
--the column list needs to be dynamic forcing this whole line to be dynamic
_t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\transsearchcsv.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c);
_t := format('COPY csv_i (%s) FROM %L WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c,_path);
--RAISE NOTICE '%', _t;
EXECUTE _t;
WITH
END
$$;
-------------for each imported row in the COPY table, genereate the json rec, and a column for the json key specified in the srce.defn-----------
WITH
-------------for each imported row in the COPY table, genereate the json rec, and a column for the json key specified in the srce.defn-----------
pending_list AS (
pending_list AS (
SELECT
---creates a key value pair and then aggregates rows of key value pairs
jsonb_object_agg(
@ -77,7 +77,7 @@ pending_list AS (
FROM
csv_i i
INNER JOIN tps.srce s ON
s.srce = 'PNCC'
s.srce = _srce
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE
GROUP BY
i.*,
@ -85,38 +85,39 @@ pending_list AS (
id
ORDER BY
id ASC
)
)
-----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
-----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
, pending_keys AS (
, pending_keys AS (
SELECT DISTINCT
json_key
FROM
pending_list
)
)
-----------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
-----------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
, unmatched_keys AS (
SELECT
, unmatched_keys AS (
SELECT
json_key
FROM
FROM
pending_keys
EXCEPT
EXCEPT
SELECT DISTINCT
SELECT DISTINCT
k.json_key
FROM
FROM
pending_keys k
INNER JOIN tps.trans t ON
t.rec @> k.json_key
)
)
-----------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
-----------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
--need to look into mapping the transactions prior to loading
, inserted AS (
, inserted AS (
INSERT INTO
tps.trans (srce, rec)
SELECT
@ -131,11 +132,11 @@ FROM
----this conflict is only if an exact duplicate rec json happens, which will be rejected
----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified
RETURNING *
)
)
-----------list of records not inserted--------------------------------------------------------------------------------------------------------------
-----------list of records not inserted--------------------------------------------------------------------------------------------------------------
, not_inserted AS (
, not_inserted AS (
SELECT
srce
,rec
@ -149,21 +150,29 @@ FROM
,rec
FROM
inserted
)
)
--------summarize records not inserted-------------------+------------------------------------------------------------------------------------------------
--------insert to log-------------------------------------------------------------------------------------------------------------------------------------
--below select should be loaded to the log table
SELECT
--------summarize records not inserted-------------------+------------------------------------------------------------------------------------------------
SELECT
t.srce
,(ae.e::text[])[1] unq_constr
,MIN(rec #>> ae.e::text[]) min_text
,MAX(rec #>> ae.e::text[]) max_text
,JSONB_PRETTY(JSON_AGG(rec #> ae.e::text[] ORDER BY rec #>> ae.e::text[])::JSONB)
FROM
FROM
not_inserted t
INNER JOIN tps.srce s ON
s.srce = t.srce
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE
GROUP BY
GROUP BY
t.srce
,(ae.e::text[])[1];
END
$$;