push inserts into do block so they can consume local variables, need to setup log inserts

srce.pgsql

@@ -15,9 +15,14 @@ DO $$
 
 DECLARE _t text;
 DECLARE _c text;
+DECLARE _path text;
+DECLARE _srce text;
 
 BEGIN
 
+    _path := 'C:\users\ptrowbridge\downloads\lon_loan_ledgerbal.csv';
+    _srce := 'PNCO';
+	
 ----------------------------------------------------build the column list of the temp table----------------------------------------------------------------
 
 	SELECT
@@ -31,7 +36,7 @@ BEGIN
         --unwrap the schema definition array
         LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE
     WHERE   
-        srce = 'PNCC'
+        srce = _srce
     GROUP BY
         srce;
         
@@ -48,22 +53,17 @@ BEGIN
 ----------------------------------------------------do the insert-------------------------------------------------------------------------------------------
 
     --the column list needs to be dynamic forcing this whole line to be dynamic
-    _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\transsearchcsv.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c);
+    _t := format('COPY csv_i (%s) FROM %L WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c,_path);
 
     --RAISE NOTICE '%', _t;
 
     EXECUTE _t;
 
+    WITH 
 
-END
+    -------------for each imported row in the COPY table, genereate the json rec, and a column for the json key specified in the srce.defn-----------
-$$;
 
-
+    pending_list AS (
-WITH 
-
--------------for each imported row in the COPY table, genereate the json rec, and a column for the json key specified in the srce.defn-----------
-
-pending_list AS (
         SELECT
             ---creates a key value pair and then aggregates rows of key value pairs
             jsonb_object_agg(
@@ -77,7 +77,7 @@ pending_list AS (
         FROM
             csv_i i
             INNER JOIN tps.srce s ON
-            s.srce = 'PNCC'
+                s.srce = _srce
             LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE
         GROUP BY
             i.*,
@@ -85,38 +85,39 @@ pending_list AS (
             id
         ORDER BY    
             id ASC
-)
+    )
 
------------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
+    -----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
 
-, pending_keys AS (
+    , pending_keys AS (
         SELECT DISTINCT
             json_key
         FROM 
             pending_list
-)
+    )
 
------------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
+    -----------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
 
-, unmatched_keys AS (
+    , unmatched_keys AS (
-SELECT
+    SELECT
         json_key
-FROM
+    FROM
         pending_keys
 
-EXCEPT
+    EXCEPT
 
-SELECT DISTINCT
+    SELECT DISTINCT
         k.json_key
-FROM
+    FROM
         pending_keys k
         INNER JOIN tps.trans t ON
             t.rec @> k.json_key
-)
+    )
 
------------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
+    -----------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
+    --need to look into mapping the transactions prior to loading
 
-, inserted AS (
+    , inserted AS (
         INSERT INTO
             tps.trans (srce, rec)
         SELECT
@@ -131,11 +132,11 @@ FROM
         ----this conflict is only if an exact duplicate rec json happens, which will be rejected
         ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified
         RETURNING *
-)
+    )
 
------------list of records not inserted--------------------------------------------------------------------------------------------------------------
+    -----------list of records not inserted--------------------------------------------------------------------------------------------------------------
 
-, not_inserted AS (
+    , not_inserted AS (
         SELECT
             srce
             ,rec
@@ -149,21 +150,29 @@ FROM
             ,rec
         FROM 
             inserted
-)
+    )
 
---------summarize records not inserted-------------------+------------------------------------------------------------------------------------------------
+    --------insert to log-------------------------------------------------------------------------------------------------------------------------------------
+    --below select should be loaded to the log table
 
-SELECT
+
+
+    --------summarize records not inserted-------------------+------------------------------------------------------------------------------------------------
+
+    SELECT
         t.srce
         ,(ae.e::text[])[1] unq_constr
         ,MIN(rec #>> ae.e::text[]) min_text
         ,MAX(rec #>> ae.e::text[]) max_text
         ,JSONB_PRETTY(JSON_AGG(rec #> ae.e::text[] ORDER BY rec #>> ae.e::text[])::JSONB)
-FROM
+    FROM
         not_inserted t
         INNER JOIN tps.srce s ON
             s.srce = t.srce
         LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE
-GROUP BY
+    GROUP BY
         t.srce
         ,(ae.e::text[])[1];
+
+END
+$$;