pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

retain the id number in the json rec, and order by id asc before inserting to trans. if mult records per key, this helps sort

Browse Source
This commit is contained in:
Paul Trowbridge 2017-10-19 11:22:03 -04:00
parent 2190f55acb
commit f0838b28c0
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
srce.pgsql
View File

@ -18,7 +18,7 @@ BEGIN
--unwrap the schema definition array --unwrap the schema definition array
LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE
WHERE WHERE
srce = 'PNCC' srce = 'PNCL'
GROUP BY GROUP BY
srce; srce;
@ -35,7 +35,7 @@ BEGIN
----------------------------------------------------do the insert------------------------------------------------------------------------------------------- ----------------------------------------------------do the insert-------------------------------------------------------------------------------------------
--the column list needs to be dynamic forcing this whole line to be dynamic --the column list needs to be dynamic forcing this whole line to be dynamic
_t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\transsearchcsv.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\llcol.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c);
--RAISE NOTICE '%', _t; --RAISE NOTICE '%', _t;
@ -55,21 +55,21 @@ WITH pending_list AS (
(ae.e::text[])[1], --the key name (ae.e::text[])[1], --the key name
(row_to_json(i)::jsonb) #> ae.e::text[] --get the target value from the key from the csv row that has been converted to json (row_to_json(i)::jsonb) #> ae.e::text[] --get the target value from the key from the csv row that has been converted to json
) json_key, ) json_key,
row_to_json(i)::JSONB - 'id' rec, row_to_json(i)::JSONB rec,
srce, srce,
--ae.rn, --ae.rn,
id id
FROM FROM
csv_i i csv_i i
INNER JOIN tps.srce s ON INNER JOIN tps.srce s ON
s.srce = 'PNCC' s.srce = 'PNCL'
LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE
GROUP BY GROUP BY
i.*, i.*,
srce, srce,
id id
ORDER BY ORDER BY
id id ASC
) )
------results of an insert operation-------------- ------results of an insert operation--------------
, inserted AS ( , inserted AS (
@ -85,6 +85,8 @@ WITH pending_list AS (
AND t.rec @> pl.json_key AND t.rec @> pl.json_key
WHERE WHERE
t IS NULL t IS NULL
ORDER BY
id ASC
----this conflict is only if an exact duplicate rec json happens, which will be rejected ----this conflict is only if an exact duplicate rec json happens, which will be rejected
----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified
RETURNING * RETURNING *