diff --git a/config/modules/poh.json b/config/modules/poh.json new file mode 100644 index 0000000..eb6816c --- /dev/null +++ b/config/modules/poh.json @@ -0,0 +1,694 @@ +{ + "name": "poh", + "source_connection": "S78030956", + "dest_connection": "usmidsap02", + "dest_table": "cms.poh", + "staging_table": "pipekit_staging.poh", + "merge_strategy": "incremental", + "merge_key": "kapo#", + "enabled": 1, + "dest_description": "Purchase Order Header 6.1", + "columns": [ + { + "source_name": "KAPO#", + "source_type": "NUMERIC(9,0)", + "dest_name": "kapo#", + "dest_type": "numeric(9,0)", + "description": "Purchase Order Number" + }, + { + "source_name": "KAOTYP", + "source_type": "CHAR(1)", + "dest_name": "kaotyp", + "dest_type": "text", + "description": "S-Standard, B-Blanket" + }, + { + "source_name": "KARELD", + "source_type": "CHAR(1)", + "dest_name": "kareld", + "dest_type": "text", + "description": "Released" + }, + { + "source_name": "KAODAT", + "source_type": "DATE", + "dest_name": "kaodat", + "dest_type": "date", + "description": "Order Date" + }, + { + "source_name": "KAOVND", + "source_type": "CHAR(10)", + "dest_name": "kaovnd", + "dest_type": "text", + "description": "Vendor Code" + }, + { + "source_name": "KAOVNM", + "source_type": "CHAR(30)", + "dest_name": "kaovnm", + "dest_type": "text", + "description": "Vendor Name" + }, + { + "source_name": "KABVND", + "source_type": "CHAR(10)", + "dest_name": "kabvnd", + "dest_type": "text", + "description": "RemitTo Vendor Number" + }, + { + "source_name": "KASVND", + "source_type": "CHAR(10)", + "dest_name": "kasvnd", + "dest_type": "text", + "description": "Shipto Vendor Number" + }, + { + "source_name": "KASVNM", + "source_type": "CHAR(30)", + "dest_name": "kasvnm", + "dest_type": "text", + "description": "Shipto Vendor Name" + }, + { + "source_name": "KASAD1", + "source_type": "CHAR(35)", + "dest_name": "kasad1", + "dest_type": "text", + "description": "Shipto Vendor Address1" + }, + { + "source_name": "KASAD2", + "source_type": "CHAR(35)", + "dest_name": "kasad2", + "dest_type": "text", + "description": "Shipto Vendor Address2" + }, + { + "source_name": "KASAD3", + "source_type": "CHAR(35)", + "dest_name": "kasad3", + "dest_type": "text", + "description": "Shipto Vendor Address3" + }, + { + "source_name": "KASAD4", + "source_type": "CHAR(35)", + "dest_name": "kasad4", + "dest_type": "text", + "description": "Shipto Vendor Address4" + }, + { + "source_name": "KASAD5", + "source_type": "CHAR(35)", + "dest_name": "kasad5", + "dest_type": "text", + "description": "Shipto Vendor Address5" + }, + { + "source_name": "KASAD6", + "source_type": "CHAR(35)", + "dest_name": "kasad6", + "dest_type": "text", + "description": "Shipto Vendor Address6" + }, + { + "source_name": "KASAD7", + "source_type": "CHAR(35)", + "dest_name": "kasad7", + "dest_type": "text", + "description": "Shipto Vendor Address7" + }, + { + "source_name": "KASAD8", + "source_type": "CHAR(35)", + "dest_name": "kasad8", + "dest_type": "text", + "description": "Shipto Vendor Address8" + }, + { + "source_name": "KASAD9", + "source_type": "CHAR(35)", + "dest_name": "kasad9", + "dest_type": "text", + "description": "Shipto Vendor Address9" + }, + { + "source_name": "KASAD10", + "source_type": "CHAR(35)", + "dest_name": "kasad10", + "dest_type": "text", + "description": "Shipto Vendor Addres10" + }, + { + "source_name": "KASVPC", + "source_type": "CHAR(10)", + "dest_name": "kasvpc", + "dest_type": "text", + "description": "Shipto Postal Code" + }, + { + "source_name": "KACNTC", + "source_type": "CHAR(20)", + "dest_name": "kacntc", + "dest_type": "text", + "description": "Contact Person" + }, + { + "source_name": "KAPHON", + "source_type": "CHAR(20)", + "dest_name": "kaphon", + "dest_type": "text", + "description": "Phone Number" + }, + { + "source_name": "KATRMC", + "source_type": "CHAR(2)", + "dest_name": "katrmc", + "dest_type": "text", + "description": "Terms Code" + }, + { + "source_name": "KASVIA", + "source_type": "CHAR(30)", + "dest_name": "kasvia", + "dest_type": "text", + "description": "Ship Via" + }, + { + "source_name": "KAPCOL", + "source_type": "CHAR(1)", + "dest_name": "kapcol", + "dest_type": "text", + "description": "P-Prepaid,C-Collect,I-Charge" + }, + { + "source_name": "KACONF", + "source_type": "CHAR(1)", + "dest_name": "kaconf", + "dest_type": "text", + "description": "Confirmation" + }, + { + "source_name": "KAFOBC", + "source_type": "CHAR(3)", + "dest_name": "kafobc", + "dest_type": "text", + "description": "FOB Code" + }, + { + "source_name": "KAOSTS", + "source_type": "CHAR(1)", + "dest_name": "kaosts", + "dest_type": "text", + "description": "Status - N-New,P-Printed,B-Backordered,C-Complete" + }, + { + "source_name": "KAPSTS", + "source_type": "CHAR(1)", + "dest_name": "kapsts", + "dest_type": "text", + "description": "Print Status" + }, + { + "source_name": "KAOGST", + "source_type": "DECIMAL(11,2)", + "dest_name": "kaogst", + "dest_type": "numeric(11,2)", + "description": "Recoverable Tax Amount" + }, + { + "source_name": "KAOPST", + "source_type": "DECIMAL(11,2)", + "dest_name": "kaopst", + "dest_type": "numeric(11,2)", + "description": "Non-Recov Tax Amount" + }, + { + "source_name": "KAOTOT", + "source_type": "DECIMAL(11,2)", + "dest_name": "kaotot", + "dest_type": "numeric(11,2)", + "description": "Order Total" + }, + { + "source_name": "KABVAL", + "source_type": "DECIMAL(11,2)", + "dest_name": "kabval", + "dest_type": "numeric(11,2)", + "description": "Blanket Value" + }, + { + "source_name": "KACURR", + "source_type": "CHAR(2)", + "dest_name": "kacurr", + "dest_type": "text", + "description": "Currency Code" + }, + { + "source_name": "KABUYR", + "source_type": "CHAR(10)", + "dest_name": "kabuyr", + "dest_type": "text", + "description": "Buyer Code" + }, + { + "source_name": "KALRL#", + "source_type": "NUMERIC(4,0)", + "dest_name": "kalrl#", + "dest_type": "numeric(4,0)", + "description": "Last Release Number" + }, + { + "source_name": "KAQNUM", + "source_type": "NUMERIC(9,0)", + "dest_name": "kaqnum", + "dest_type": "numeric(9,0)", + "description": "P/O Quote Number" + }, + { + "source_name": "KACRCM", + "source_type": "CHAR(1)", + "dest_name": "kacrcm", + "dest_type": "text", + "description": "1-Current,2-Complete" + }, + { + "source_name": "KACARC", + "source_type": "CHAR(17)", + "dest_name": "kacarc", + "dest_type": "text", + "description": "Carrier Code" + }, + { + "source_name": "KAFRAM", + "source_type": "DECIMAL(11,2)", + "dest_name": "kafram", + "dest_type": "numeric(11,2)", + "description": "Freight Amount" + }, + { + "source_name": "KACBRC", + "source_type": "CHAR(10)", + "dest_name": "kacbrc", + "dest_type": "text", + "description": "Customs Broker Code" + }, + { + "source_name": "KADUAM", + "source_type": "DECIMAL(11,2)", + "dest_name": "kaduam", + "dest_type": "numeric(11,2)", + "description": "Duty & Brokerage Fee" + }, + { + "source_name": "KAHREA", + "source_type": "CHAR(1)", + "dest_name": "kahrea", + "dest_type": "text", + "description": "Hold Reason Code" + }, + { + "source_name": "KAFAX", + "source_type": "CHAR(20)", + "dest_name": "kafax", + "dest_type": "text", + "description": "Fax Number" + }, + { + "source_name": "KAIUSE", + "source_type": "CHAR(1)", + "dest_name": "kaiuse", + "dest_type": "text", + "description": "IN USE FLAG" + }, + { + "source_name": "KAPURP", + "source_type": "CHAR(1)", + "dest_name": "kapurp", + "dest_type": "text", + "description": "Tran Purpse Code" + }, + { + "source_name": "KAESTS", + "source_type": "CHAR(1)", + "dest_name": "kaests", + "dest_type": "text", + "description": "EDI Send Status" + }, + { + "source_name": "KAASTS", + "source_type": "CHAR(1)", + "dest_name": "kaasts", + "dest_type": "text", + "description": "EDI Acknowledge Status" + }, + { + "source_name": "KAPLNT", + "source_type": "CHAR(3)", + "dest_name": "kaplnt", + "dest_type": "text", + "description": "Plant Code" + }, + { + "source_name": "KASCTY", + "source_type": "CHAR(30)", + "dest_name": "kascty", + "dest_type": "text", + "description": "Shipto City" + }, + { + "source_name": "KASPOV", + "source_type": "CHAR(3)", + "dest_name": "kaspov", + "dest_type": "text", + "description": "Shipto State/Province" + }, + { + "source_name": "KASCRY", + "source_type": "CHAR(3)", + "dest_name": "kascry", + "dest_type": "text", + "description": "Shipto Country" + }, + { + "source_name": "KAFUT1", + "source_type": "CHAR(1)", + "dest_name": "kafut1", + "dest_type": "text", + "description": "Future Use" + }, + { + "source_name": "KAFUT2", + "source_type": "CHAR(1)", + "dest_name": "kafut2", + "dest_type": "text", + "description": "Ever Maintain" + }, + { + "source_name": "KAFUT3", + "source_type": "CHAR(1)", + "dest_name": "kafut3", + "dest_type": "text", + "description": "Web Service In-Use" + }, + { + "source_name": "KAFUT4", + "source_type": "CHAR(1)", + "dest_name": "kafut4", + "dest_type": "text", + "description": "FUTURE 4" + }, + { + "source_name": "KAFUT5", + "source_type": "CHAR(1)", + "dest_name": "kafut5", + "dest_type": "text", + "description": "FUTURE 5" + }, + { + "source_name": "KAFUT6", + "source_type": "CHAR(1)", + "dest_name": "kafut6", + "dest_type": "text", + "description": "FUTURE 6" + }, + { + "source_name": "KAFUT7", + "source_type": "CHAR(1)", + "dest_name": "kafut7", + "dest_type": "text", + "description": "FUTURE 7" + }, + { + "source_name": "KAFUT8", + "source_type": "CHAR(1)", + "dest_name": "kafut8", + "dest_type": "text", + "description": "FUTURE 8" + }, + { + "source_name": "KAFUT9", + "source_type": "CHAR(1)", + "dest_name": "kafut9", + "dest_type": "text", + "description": "FUTURE 9" + }, + { + "source_name": "KAFUT10", + "source_type": "CHAR(1)", + "dest_name": "kafut10", + "dest_type": "text", + "description": "FUTURE 10" + }, + { + "source_name": "KAFUT11", + "source_type": "CHAR(10)", + "dest_name": "kafut11", + "dest_type": "text", + "description": "Created By User" + }, + { + "source_name": "KAFUT12", + "source_type": "CHAR(10)", + "dest_name": "kafut12", + "dest_type": "text", + "description": "Date Created" + }, + { + "source_name": "KAFUT13", + "source_type": "CHAR(10)", + "dest_name": "kafut13", + "dest_type": "text", + "description": "Time Created" + }, + { + "source_name": "KAFUT14", + "source_type": "CHAR(10)", + "dest_name": "kafut14", + "dest_type": "text", + "description": "FUTURE 14" + }, + { + "source_name": "KAFUT15", + "source_type": "CHAR(10)", + "dest_name": "kafut15", + "dest_type": "text", + "description": "FUTURE 15" + }, + { + "source_name": "KAFUT16", + "source_type": "CHAR(10)", + "dest_name": "kafut16", + "dest_type": "text", + "description": "Last Aprvl/Reject Date" + }, + { + "source_name": "KAFUT17", + "source_type": "CHAR(10)", + "dest_name": "kafut17", + "dest_type": "text", + "description": "Last Aprvl/Reject Time" + }, + { + "source_name": "KAFUT18", + "source_type": "CHAR(10)", + "dest_name": "kafut18", + "dest_type": "text", + "description": "FUTURE 18" + }, + { + "source_name": "KAFUT19", + "source_type": "CHAR(10)", + "dest_name": "kafut19", + "dest_type": "text", + "description": "Last Approval Checking Date" + }, + { + "source_name": "KAFUT20", + "source_type": "CHAR(10)", + "dest_name": "kafut20", + "dest_type": "text", + "description": "Date Updated" + }, + { + "source_name": "KAFUT21", + "source_type": "CHAR(20)", + "dest_name": "kafut21", + "dest_type": "text", + "description": "Time Updated" + }, + { + "source_name": "KAFUT22", + "source_type": "CHAR(20)", + "dest_name": "kafut22", + "dest_type": "text", + "description": "Last Approval Checking Time" + }, + { + "source_name": "KAFUT23", + "source_type": "CHAR(20)", + "dest_name": "kafut23", + "dest_type": "text", + "description": "FUTURE 23" + }, + { + "source_name": "KAFUT24", + "source_type": "CHAR(20)", + "dest_name": "kafut24", + "dest_type": "text", + "description": "FUTURE 24" + }, + { + "source_name": "KAFUT25", + "source_type": "CHAR(20)", + "dest_name": "kafut25", + "dest_type": "text", + "description": "FUTURE 25" + }, + { + "source_name": "KAFUT26", + "source_type": "CHAR(21)", + "dest_name": "kafut26", + "dest_type": "text", + "description": "FUTURE 26" + }, + { + "source_name": "KAFUT27", + "source_type": "CHAR(22)", + "dest_name": "kafut27", + "dest_type": "text", + "description": "FUTURE 27" + }, + { + "source_name": "KAFUT28", + "source_type": "CHAR(23)", + "dest_name": "kafut28", + "dest_type": "text", + "description": "FUTURE 28" + }, + { + "source_name": "KAFUT29", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut29", + "dest_type": "numeric(15,5)", + "description": "FUTURE 29" + }, + { + "source_name": "KAFUT30", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut30", + "dest_type": "numeric(15,5)", + "description": "FUTURE 30" + }, + { + "source_name": "KAFUT31", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut31", + "dest_type": "numeric(15,5)", + "description": "FUTURE 31" + }, + { + "source_name": "KAFUT32", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut32", + "dest_type": "numeric(15,5)", + "description": "FUTURE 32" + }, + { + "source_name": "KAFUT33", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut33", + "dest_type": "numeric(15,5)", + "description": "FUTURE 33" + }, + { + "source_name": "KAFUT34", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut34", + "dest_type": "numeric(15,5)", + "description": "FUTURE 34" + }, + { + "source_name": "KAFUT35", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut35", + "dest_type": "numeric(15,5)", + "description": "FUTURE 35" + }, + { + "source_name": "KAFUT36", + "source_type": "NUMERIC(15,5)", + "dest_name": "kafut36", + "dest_type": "numeric(15,5)", + "description": "FUTURE 36" + }, + { + "source_name": "KAUSR1", + "source_type": "CHAR(10)", + "dest_name": "kausr1", + "dest_type": "text", + "description": "User 1" + }, + { + "source_name": "KAUSR2", + "source_type": "CHAR(10)", + "dest_name": "kausr2", + "dest_type": "text", + "description": "User 2" + }, + { + "source_name": "KAUSR3", + "source_type": "CHAR(10)", + "dest_name": "kausr3", + "dest_type": "text", + "description": "User 3" + }, + { + "source_name": "KAUSR4", + "source_type": "CHAR(10)", + "dest_name": "kausr4", + "dest_type": "text", + "description": "User 4" + }, + { + "source_name": "KAUSR5", + "source_type": "CHAR(10)", + "dest_name": "kausr5", + "dest_type": "text", + "description": "User 5" + }, + { + "source_name": "KAUSR6", + "source_type": "CHAR(10)", + "dest_name": "kausr6", + "dest_type": "text", + "description": "User 6" + }, + { + "source_name": "KAUSR7", + "source_type": "CHAR(10)", + "dest_name": "kausr7", + "dest_type": "text", + "description": "User 7" + }, + { + "source_name": "KAEDIRS", + "source_type": "CHAR(1)", + "dest_name": "kaedirs", + "dest_type": "text", + "description": "EDI Re-send" + } + ], + "watermarks": [ + { + "name": "poh_wm", + "connection": "usmidsap02", + "resolver_sql": "SELECT TO_CHAR(LEAST(COALESCE(MAX(NULLIF(kafut20, ''))::DATE, DATE '1900-01-01'), CURRENT_DATE) - 7, 'YYYY-MM-DD') FROM cms.poh", + "default_value": null + } + ], + "hooks": [] +} diff --git a/config/modules/poh.sql b/config/modules/poh.sql new file mode 100644 index 0000000..e51b897 --- /dev/null +++ b/config/modules/poh.sql @@ -0,0 +1,108 @@ +SELECT + "KAPO#" AS "kapo#", + RTRIM(KAOTYP) AS kaotyp, + RTRIM(KARELD) AS kareld, + CASE WHEN KAODAT IN (DATE('0001-01-01'), DATE('9999-12-31')) THEN NULL ELSE KAODAT END AS kaodat, + RTRIM(KAOVND) AS kaovnd, + RTRIM(KAOVNM) AS kaovnm, + RTRIM(KABVND) AS kabvnd, + RTRIM(KASVND) AS kasvnd, + RTRIM(KASVNM) AS kasvnm, + RTRIM(KASAD1) AS kasad1, + RTRIM(KASAD2) AS kasad2, + RTRIM(KASAD3) AS kasad3, + RTRIM(KASAD4) AS kasad4, + RTRIM(KASAD5) AS kasad5, + RTRIM(KASAD6) AS kasad6, + RTRIM(KASAD7) AS kasad7, + RTRIM(KASAD8) AS kasad8, + RTRIM(KASAD9) AS kasad9, + RTRIM(KASAD10) AS kasad10, + RTRIM(KASVPC) AS kasvpc, + RTRIM(KACNTC) AS kacntc, + RTRIM(KAPHON) AS kaphon, + RTRIM(KATRMC) AS katrmc, + RTRIM(KASVIA) AS kasvia, + RTRIM(KAPCOL) AS kapcol, + RTRIM(KACONF) AS kaconf, + RTRIM(KAFOBC) AS kafobc, + RTRIM(KAOSTS) AS kaosts, + RTRIM(KAPSTS) AS kapsts, + KAOGST AS kaogst, + KAOPST AS kaopst, + KAOTOT AS kaotot, + KABVAL AS kabval, + RTRIM(KACURR) AS kacurr, + RTRIM(KABUYR) AS kabuyr, + "KALRL#" AS "kalrl#", + KAQNUM AS kaqnum, + RTRIM(KACRCM) AS kacrcm, + RTRIM(KACARC) AS kacarc, + KAFRAM AS kafram, + RTRIM(KACBRC) AS kacbrc, + KADUAM AS kaduam, + RTRIM(KAHREA) AS kahrea, + RTRIM(KAFAX) AS kafax, + RTRIM(KAIUSE) AS kaiuse, + RTRIM(KAPURP) AS kapurp, + RTRIM(KAESTS) AS kaests, + RTRIM(KAASTS) AS kaasts, + RTRIM(KAPLNT) AS kaplnt, + RTRIM(KASCTY) AS kascty, + RTRIM(KASPOV) AS kaspov, + RTRIM(KASCRY) AS kascry, + RTRIM(KAFUT1) AS kafut1, + RTRIM(KAFUT2) AS kafut2, + RTRIM(KAFUT3) AS kafut3, + RTRIM(KAFUT4) AS kafut4, + RTRIM(KAFUT5) AS kafut5, + RTRIM(KAFUT6) AS kafut6, + RTRIM(KAFUT7) AS kafut7, + RTRIM(KAFUT8) AS kafut8, + RTRIM(KAFUT9) AS kafut9, + RTRIM(KAFUT10) AS kafut10, + RTRIM(KAFUT11) AS kafut11, + RTRIM(KAFUT12) AS kafut12, + RTRIM(KAFUT13) AS kafut13, + RTRIM(KAFUT14) AS kafut14, + RTRIM(KAFUT15) AS kafut15, + RTRIM(KAFUT16) AS kafut16, + RTRIM(KAFUT17) AS kafut17, + RTRIM(KAFUT18) AS kafut18, + RTRIM(KAFUT19) AS kafut19, + RTRIM(KAFUT20) AS kafut20, + RTRIM(KAFUT21) AS kafut21, + RTRIM(KAFUT22) AS kafut22, + RTRIM(KAFUT23) AS kafut23, + RTRIM(KAFUT24) AS kafut24, + RTRIM(KAFUT25) AS kafut25, + RTRIM(KAFUT26) AS kafut26, + RTRIM(KAFUT27) AS kafut27, + RTRIM(KAFUT28) AS kafut28, + KAFUT29 AS kafut29, + KAFUT30 AS kafut30, + KAFUT31 AS kafut31, + KAFUT32 AS kafut32, + KAFUT33 AS kafut33, + KAFUT34 AS kafut34, + KAFUT35 AS kafut35, + KAFUT36 AS kafut36, + RTRIM(KAUSR1) AS kausr1, + RTRIM(KAUSR2) AS kausr2, + RTRIM(KAUSR3) AS kausr3, + RTRIM(KAUSR4) AS kausr4, + RTRIM(KAUSR5) AS kausr5, + RTRIM(KAUSR6) AS kausr6, + RTRIM(KAUSR7) AS kausr7, + RTRIM(KAEDIRS) AS kaedirs +FROM LGDAT.POH +-- DB2 for i has no boolean literal, so no `WHERE TRUE` base condition here. +-- POH's "future" fields are really an audit pair: KAFUT12 = Date Created, +-- KAFUT20 = Date Updated, both CHAR(10) ISO 'YYYY-MM-DD', so the compares +-- below are lexical and blanks sort below any watermark. +WHERE ( KAFUT20 >= '{poh_wm}' + OR KAFUT12 >= '{poh_wm}' + -- Open-PO hot set: KACRCM 1-Current / 2-Complete, and it leads most of + -- POH's logicals, so this is a keyed read of ~1,035 rows. Insurance + -- against a status flip that writes no date, the way OCRH's DCSTAT does. + OR KACRCM = '1') diff --git a/config/modules/poh_deleted.json b/config/modules/poh_deleted.json new file mode 100644 index 0000000..4d57af8 --- /dev/null +++ b/config/modules/poh_deleted.json @@ -0,0 +1,57 @@ +{ + "name": "poh_deleted", + "source_connection": "S78030956", + "dest_connection": "usmidsap02", + "dest_table": "cms.poh_deleted", + "staging_table": "pipekit_staging.poh_deleted", + "merge_strategy": "incremental", + "merge_key": "dq0po#", + "enabled": 1, + "dest_description": "POs purged from LGDAT.POH (POHL delete log) -- drives purge cleanup of cms.poh / cms.poi", + "columns": [ + { + "id": "c1", + "source_name": "DQ0PO#", + "source_type": "NUMERIC(9,0)", + "dest_name": "dq0po#", + "dest_type": "numeric(9,0)", + "description": "Purchase Order Number" + }, + { + "id": "c2", + "source_name": "DQ0TMSP", + "source_type": "CHAR(26)", + "dest_name": "dq0tmsp", + "dest_type": "text", + "description": "Change TimeStamp of the delete" + } + ], + "watermarks": [ + { + "name": "poh_deleted_wm", + "connection": "usmidsap02", + "resolver_sql": "SELECT TO_CHAR(COALESCE(MAX(SUBSTRING(dq0tmsp, 1, 10))::DATE, DATE '1900-01-01') - 7, 'YYYY-MM-DD') FROM cms.poh_deleted", + "default_value": null + } + ], + "hooks": [ + { + "run_order": 1, + "connection": "usmidsap02", + "sql": "DELETE FROM cms.poh_deleted WHERE \"dq0po#\" IN (SELECT \"kapo#\" FROM cms.poh)", + "run_on": "success" + }, + { + "run_order": 2, + "connection": "usmidsap02", + "sql": "DELETE FROM cms.poh WHERE \"kapo#\" IN (SELECT \"dq0po#\" FROM cms.poh_deleted)", + "run_on": "success" + }, + { + "run_order": 3, + "connection": "usmidsap02", + "sql": "DELETE FROM cms.poi WHERE \"kbpo#\" IN (SELECT \"dq0po#\" FROM cms.poh_deleted)", + "run_on": "success" + } + ] +} diff --git a/config/modules/poh_deleted.sql b/config/modules/poh_deleted.sql new file mode 100644 index 0000000..15c95da --- /dev/null +++ b/config/modules/poh_deleted.sql @@ -0,0 +1,20 @@ +-- POs purged from LGDAT.POH. An incremental merge only deletes keys that are +-- present in staging, so a purged PO would sit in cms.poh / cms.poi forever; +-- this list drives the hooks below that remove them. +-- +-- DQ0ACTN = '3' is the delete action (verified 2026-08-18: PO 618973's last log +-- event was an action 3 and the row is gone from POH). The NOT EXISTS is what +-- makes this exact rather than a guess about action ordering -- a PO number that +-- was deleted and later re-used is back in POH, so it drops out of the list on +-- its own, and hook 1 clears any tombstone left from an earlier run. +-- +-- DQ0TMSP is CHAR(26) 'YYYY-MM-DD-HH.MM.SS.ffffff', so the watermark compare is +-- lexical, and POHL has a timestamp-only logical (POHLX1) to range-scan it. +SELECT + l."DQ0PO#" AS "dq0po#", + MAX(l.DQ0TMSP) AS dq0tmsp +FROM LGDAT.POHL l +WHERE l.DQ0ACTN = '3' + AND l.DQ0TMSP >= '{poh_deleted_wm}' + AND NOT EXISTS (SELECT 1 FROM LGDAT.POH p WHERE p."KAPO#" = l."DQ0PO#") +GROUP BY l."DQ0PO#" diff --git a/config/modules/poi.json b/config/modules/poi.json new file mode 100644 index 0000000..30c2e61 --- /dev/null +++ b/config/modules/poi.json @@ -0,0 +1,666 @@ +{ + "name": "poi", + "source_connection": "S78030956", + "dest_connection": "usmidsap02", + "dest_table": "cms.poi", + "staging_table": "pipekit_staging.poi", + "merge_strategy": "incremental", + "merge_key": "kbpo#", + "enabled": 1, + "dest_description": "Purchase Order Items 6.0", + "columns": [ + { + "source_name": "KBPO#", + "source_type": "NUMERIC(9,0)", + "dest_name": "kbpo#", + "dest_type": "numeric(9,0)", + "description": "Purchase Order Number" + }, + { + "source_name": "KBITM#", + "source_type": "NUMERIC(3,0)", + "dest_name": "kbitm#", + "dest_type": "numeric(3,0)", + "description": "P/O Item #" + }, + { + "source_name": "KBQTYO", + "source_type": "DECIMAL(11,2)", + "dest_name": "kbqtyo", + "dest_type": "numeric(11,2)", + "description": "Quantity Ordered" + }, + { + "source_name": "KBQTYR", + "source_type": "DECIMAL(15,5)", + "dest_name": "kbqtyr", + "dest_type": "numeric(15,5)", + "description": "Quantity Received" + }, + { + "source_name": "KBQTYA", + "source_type": "DECIMAL(15,5)", + "dest_name": "kbqtya", + "dest_type": "numeric(15,5)", + "description": "Quantity Approved" + }, + { + "source_name": "KBQTYJ", + "source_type": "DECIMAL(15,5)", + "dest_name": "kbqtyj", + "dest_type": "numeric(15,5)", + "description": "Quantity Rejected" + }, + { + "source_name": "KBOUNT", + "source_type": "CHAR(3)", + "dest_name": "kbount", + "dest_type": "text", + "description": "Order Unit" + }, + { + "source_name": "KBPT#", + "source_type": "CHAR(20)", + "dest_name": "kbpt#", + "dest_type": "text", + "description": "Part Number" + }, + { + "source_name": "KBVPT#", + "source_type": "CHAR(30)", + "dest_name": "kbvpt#", + "dest_type": "text", + "description": "Vendor Part Number" + }, + { + "source_name": "KBUPRC", + "source_type": "DECIMAL(13,5)", + "dest_name": "kbuprc", + "dest_type": "numeric(13,5)", + "description": "Unit Price" + }, + { + "source_name": "KBPUNT", + "source_type": "CHAR(3)", + "dest_name": "kbpunt", + "dest_type": "text", + "description": "Price Unit" + }, + { + "source_name": "KBRDAT", + "source_type": "DATE", + "dest_name": "kbrdat", + "dest_type": "date", + "description": "Date Required" + }, + { + "source_name": "KBCDAT", + "source_type": "DATE", + "dest_name": "kbcdat", + "dest_type": "date", + "description": "Date Confirmed" + }, + { + "source_name": "KBREQR", + "source_type": "CHAR(10)", + "dest_name": "kbreqr", + "dest_type": "text", + "description": "Requisitioner" + }, + { + "source_name": "KBREQ#", + "source_type": "NUMERIC(9,0)", + "dest_name": "kbreq#", + "dest_type": "numeric(9,0)", + "description": "P/O Requisition Number" + }, + { + "source_name": "KBGL#", + "source_type": "NUMERIC(10,0)", + "dest_name": "kbgl#", + "dest_type": "numeric(10,0)", + "description": "G/L Account Number" + }, + { + "source_name": "KBJOB#", + "source_type": "CHAR(10)", + "dest_name": "kbjob#", + "dest_type": "text", + "description": "Job Number" + }, + { + "source_name": "KBSEQ#", + "source_type": "NUMERIC(3,0)", + "dest_name": "kbseq#", + "dest_type": "numeric(3,0)", + "description": "Job Sequence Number" + }, + { + "source_name": "KBPRJ#", + "source_type": "CHAR(20)", + "dest_name": "kbprj#", + "dest_type": "text", + "description": "Project Number" + }, + { + "source_name": "KBDEPT", + "source_type": "CHAR(2)", + "dest_name": "kbdept", + "dest_type": "text", + "description": "Department Number" + }, + { + "source_name": "KBCGST", + "source_type": "CHAR(1)", + "dest_name": "kbcgst", + "dest_type": "text", + "description": "OBSOLETE - DO NOT USE" + }, + { + "source_name": "KBCPST", + "source_type": "CHAR(1)", + "dest_name": "kbcpst", + "dest_type": "text", + "description": "OBSOLETE - DO NOT USE" + }, + { + "source_name": "KBISTS", + "source_type": "CHAR(1)", + "dest_name": "kbists", + "dest_type": "text", + "description": "Status - O-Open,B-Backordered,C-Complete" + }, + { + "source_name": "KBITYP", + "source_type": "CHAR(1)", + "dest_name": "kbityp", + "dest_type": "text", + "description": "Type - -Normal,D-Description,O-Outside Service,S-" + }, + { + "source_name": "KBISTR", + "source_type": "CHAR(6)", + "dest_name": "kbistr", + "dest_type": "text", + "description": "Inventory Stockroom" + }, + { + "source_name": "KBOSTR", + "source_type": "CHAR(3)", + "dest_name": "kbostr", + "dest_type": "text", + "description": "OBSOLETE FIELD" + }, + { + "source_name": "KBEXTN", + "source_type": "DECIMAL(11,2)", + "dest_name": "kbextn", + "dest_type": "numeric(11,2)", + "description": "Total Including Taxes" + }, + { + "source_name": "KBGSTA", + "source_type": "DECIMAL(11,2)", + "dest_name": "kbgsta", + "dest_type": "numeric(11,2)", + "description": "Recoverable Tax Amount" + }, + { + "source_name": "KBPSTA", + "source_type": "DECIMAL(11,2)", + "dest_name": "kbpsta", + "dest_type": "numeric(11,2)", + "description": "Non-Recov Tax Amount" + }, + { + "source_name": "KBPRGR", + "source_type": "CHAR(1)", + "dest_name": "kbprgr", + "dest_type": "text", + "description": "Progressive Release Y/N" + }, + { + "source_name": "KBCRCM", + "source_type": "CHAR(1)", + "dest_name": "kbcrcm", + "dest_type": "text", + "description": "1-Current,2-Complete" + }, + { + "source_name": "KBOTYP", + "source_type": "CHAR(1)", + "dest_name": "kbotyp", + "dest_type": "text", + "description": "S-Standard, B-blanket" + }, + { + "source_name": "KBTOS", + "source_type": "CHAR(1)", + "dest_name": "kbtos", + "dest_type": "text", + "description": "Transferred Outside Service" + }, + { + "source_name": "KBTAXG", + "source_type": "CHAR(3)", + "dest_name": "kbtaxg", + "dest_type": "text", + "description": "Tax Group Code" + }, + { + "source_name": "KBTAXR", + "source_type": "CHAR(1)", + "dest_name": "kbtaxr", + "dest_type": "text", + "description": "Tax Rate Code" + }, + { + "source_name": "KBDPRC", + "source_type": "CHAR(1)", + "dest_name": "kbdprc", + "dest_type": "text", + "description": "Dynamic Price Flag" + }, + { + "source_name": "KBPLNT", + "source_type": "CHAR(3)", + "dest_name": "kbplnt", + "dest_type": "text", + "description": "Plant Code" + }, + { + "source_name": "KBFUT1", + "source_type": "CHAR(1)", + "dest_name": "kbfut1", + "dest_type": "text", + "description": "FUTURE 1" + }, + { + "source_name": "KBFUT2", + "source_type": "CHAR(1)", + "dest_name": "kbfut2", + "dest_type": "text", + "description": "FUTURE 2" + }, + { + "source_name": "KBFUT3", + "source_type": "CHAR(1)", + "dest_name": "kbfut3", + "dest_type": "text", + "description": "FUTURE 3" + }, + { + "source_name": "KBFUT4", + "source_type": "CHAR(1)", + "dest_name": "kbfut4", + "dest_type": "text", + "description": "FUTURE 4" + }, + { + "source_name": "KBFUT5", + "source_type": "CHAR(1)", + "dest_name": "kbfut5", + "dest_type": "text", + "description": "FUTURE 5" + }, + { + "source_name": "KBFUT6", + "source_type": "CHAR(1)", + "dest_name": "kbfut6", + "dest_type": "text", + "description": "FUTURE 6" + }, + { + "source_name": "KBFUT7", + "source_type": "CHAR(1)", + "dest_name": "kbfut7", + "dest_type": "text", + "description": "FUTURE 7" + }, + { + "source_name": "KBFUT8", + "source_type": "CHAR(1)", + "dest_name": "kbfut8", + "dest_type": "text", + "description": "Initialized Min. Shipment Completion %" + }, + { + "source_name": "KBFUT9", + "source_type": "CHAR(1)", + "dest_name": "kbfut9", + "dest_type": "text", + "description": "FUTURE 9" + }, + { + "source_name": "KBFUT10", + "source_type": "CHAR(1)", + "dest_name": "kbfut10", + "dest_type": "text", + "description": "Update Last Actual" + }, + { + "source_name": "KBFUT11", + "source_type": "CHAR(10)", + "dest_name": "kbfut11", + "dest_type": "text", + "description": "FUTURE 11" + }, + { + "source_name": "KBFUT12", + "source_type": "CHAR(10)", + "dest_name": "kbfut12", + "dest_type": "text", + "description": "Revision Level" + }, + { + "source_name": "KBFUT13", + "source_type": "CHAR(10)", + "dest_name": "kbfut13", + "dest_type": "text", + "description": "FUTURE 13" + }, + { + "source_name": "KBFUT14", + "source_type": "CHAR(10)", + "dest_name": "kbfut14", + "dest_type": "text", + "description": "FUTURE 14" + }, + { + "source_name": "KBFUT15", + "source_type": "CHAR(10)", + "dest_name": "kbfut15", + "dest_type": "text", + "description": "FUTURE 15" + }, + { + "source_name": "KBFUT16", + "source_type": "CHAR(10)", + "dest_name": "kbfut16", + "dest_type": "text", + "description": "FUTURE 16" + }, + { + "source_name": "KBFUT17", + "source_type": "CHAR(10)", + "dest_name": "kbfut17", + "dest_type": "text", + "description": "FUTURE 17" + }, + { + "source_name": "KBFUT18", + "source_type": "CHAR(10)", + "dest_name": "kbfut18", + "dest_type": "text", + "description": "FUTURE 18" + }, + { + "source_name": "KBFUT19", + "source_type": "CHAR(10)", + "dest_name": "kbfut19", + "dest_type": "text", + "description": "FUTURE 19" + }, + { + "source_name": "KBFUT20", + "source_type": "CHAR(10)", + "dest_name": "kbfut20", + "dest_type": "text", + "description": "FUTURE 20" + }, + { + "source_name": "KBFUT21", + "source_type": "CHAR(20)", + "dest_name": "kbfut21", + "dest_type": "text", + "description": "FUTURE 21" + }, + { + "source_name": "KBFUT22", + "source_type": "CHAR(20)", + "dest_name": "kbfut22", + "dest_type": "text", + "description": "FUTURE 22" + }, + { + "source_name": "KBFUT23", + "source_type": "CHAR(20)", + "dest_name": "kbfut23", + "dest_type": "text", + "description": "FUTURE 23" + }, + { + "source_name": "KBFUT24", + "source_type": "CHAR(20)", + "dest_name": "kbfut24", + "dest_type": "text", + "description": "FUTURE 24" + }, + { + "source_name": "KBFUT25", + "source_type": "CHAR(20)", + "dest_name": "kbfut25", + "dest_type": "text", + "description": "FUTURE 25" + }, + { + "source_name": "KBFUT26", + "source_type": "CHAR(21)", + "dest_name": "kbfut26", + "dest_type": "text", + "description": "FUTURE 26" + }, + { + "source_name": "KBFUT27", + "source_type": "CHAR(22)", + "dest_name": "kbfut27", + "dest_type": "text", + "description": "FUTURE 27" + }, + { + "source_name": "KBFUT28", + "source_type": "CHAR(23)", + "dest_name": "kbfut28", + "dest_type": "text", + "description": "FUTURE 28" + }, + { + "source_name": "KBFUT29", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut29", + "dest_type": "numeric(15,5)", + "description": "Allowed Over-shipment Percent" + }, + { + "source_name": "KBFUT30", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut30", + "dest_type": "numeric(15,5)", + "description": "Minimum Shipment Completion %" + }, + { + "source_name": "KBFUT31", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut31", + "dest_type": "numeric(15,5)", + "description": "FUTURE 31" + }, + { + "source_name": "KBFUT32", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut32", + "dest_type": "numeric(15,5)", + "description": "FUTURE 32" + }, + { + "source_name": "KBFUT33", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut33", + "dest_type": "numeric(15,5)", + "description": "FUTURE 33" + }, + { + "source_name": "KBFUT34", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut34", + "dest_type": "numeric(15,5)", + "description": "FUTURE 34" + }, + { + "source_name": "KBFUT35", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut35", + "dest_type": "numeric(15,5)", + "description": "FUTURE 35" + }, + { + "source_name": "KBFUT36", + "source_type": "NUMERIC(15,5)", + "dest_name": "kbfut36", + "dest_type": "numeric(15,5)", + "description": "FUTURE 36" + }, + { + "source_name": "KBUSR1", + "source_type": "CHAR(10)", + "dest_name": "kbusr1", + "dest_type": "text", + "description": "User 1" + }, + { + "source_name": "KBUSR2", + "source_type": "CHAR(10)", + "dest_name": "kbusr2", + "dest_type": "text", + "description": "User 2" + }, + { + "source_name": "KBUSR3", + "source_type": "CHAR(10)", + "dest_name": "kbusr3", + "dest_type": "text", + "description": "User 3" + }, + { + "source_name": "KBUSR4", + "source_type": "CHAR(10)", + "dest_name": "kbusr4", + "dest_type": "text", + "description": "User 4" + }, + { + "source_name": "KBUSR5", + "source_type": "CHAR(10)", + "dest_name": "kbusr5", + "dest_type": "text", + "description": "User 5" + }, + { + "source_name": "KBUSR6", + "source_type": "CHAR(10)", + "dest_name": "kbusr6", + "dest_type": "text", + "description": "User 6" + }, + { + "source_name": "KBUSR7", + "source_type": "CHAR(10)", + "dest_name": "kbusr7", + "dest_type": "text", + "description": "User 7" + }, + { + "source_name": "KBCORG", + "source_type": "CHAR(3)", + "dest_name": "kbcorg", + "dest_type": "text", + "description": "Country Of Origin (Code)" + }, + { + "source_name": "KBROHD", + "source_type": "CHAR(1)", + "dest_name": "kbrohd", + "dest_type": "text", + "description": "Put Receipt on Hold For Voucher Creation" + }, + { + "source_name": "KBORDN", + "source_type": "NUMERIC(9,0)", + "dest_name": "kbordn", + "dest_type": "numeric(9,0)", + "description": "Customer Order Number" + }, + { + "source_name": "KBOITM", + "source_type": "NUMERIC(3,0)", + "dest_name": "kboitm", + "dest_type": "numeric(3,0)", + "description": "Ref. Sales Ord. Item#" + }, + { + "source_name": "KBPPRT", + "source_type": "CHAR(20)", + "dest_name": "kbpprt", + "dest_type": "text", + "description": "Project Part Number" + }, + { + "source_name": "KBCSTC", + "source_type": "CHAR(2)", + "dest_name": "kbcstc", + "dest_type": "text", + "description": "Project Cost Code" + }, + { + "source_name": "KBCSTT", + "source_type": "CHAR(1)", + "dest_name": "kbcstt", + "dest_type": "text", + "description": "Project Cost Type" + }, + { + "source_name": "KBSRIN", + "source_type": "CHAR(1)", + "dest_name": "kbsrin", + "dest_type": "text", + "description": "Surcharge Indicator" + }, + { + "source_name": "KBUNTFR", + "source_type": "NUMERIC(11,2)", + "dest_name": "kbuntfr", + "dest_type": "numeric(11,2)", + "description": "Per-Unit Freight Cost" + }, + { + "source_name": "KBSRGPC", + "source_type": "NUMERIC(8,5)", + "dest_name": "kbsrgpc", + "dest_type": "numeric(8,5)", + "description": "Surcharge Percentage" + }, + { + "source_name": "KBFFRTC", + "source_type": "NUMERIC(11,2)", + "dest_name": "kbffrtc", + "dest_type": "numeric(11,2)", + "description": "Flat Freight Cost" + }, + { + "source_name": "KBPGRP", + "source_type": "CHAR(20)", + "dest_name": "kbpgrp", + "dest_type": "text", + "description": "Commodity Category Code" + } + ], + "watermarks": [ + { + "name": "poi_wm", + "connection": "usmidsap02", + "resolver_sql": "SELECT TO_CHAR(LEAST(COALESCE(MAX(NULLIF(kafut20, ''))::DATE, DATE '1900-01-01'), CURRENT_DATE) - 7, 'YYYY-MM-DD') FROM cms.poh", + "default_value": null + } + ], + "hooks": [] +} diff --git a/config/modules/poi.sql b/config/modules/poi.sql new file mode 100644 index 0000000..e1deeeb --- /dev/null +++ b/config/modules/poi.sql @@ -0,0 +1,113 @@ +WITH changed AS ( + -- Same predicate as poh (module 138) -- POI carries no change stamp of its + -- own (every KBFUT* is either blank or a business attribute), so it learns + -- of change only through its header. Verified 2026-08-18 against + -- LGDAT.POIL (CMS's own item audit log): over a 7-day window POIL reports + -- 0 POs that POHL does not already report, i.e. a line edit always stamps + -- the header. + SELECT "KAPO#" AS po FROM LGDAT.POH WHERE KAFUT20 >= '{poi_wm}' + UNION + SELECT "KAPO#" FROM LGDAT.POH WHERE KAFUT12 >= '{poi_wm}' + UNION + SELECT "KAPO#" FROM LGDAT.POH WHERE KACRCM = '1' + UNION + -- 4 open lines hang off headers already marked complete; without this they + -- would never refresh. + SELECT "KBPO#" FROM LGDAT.POI WHERE KBCRCM = '1' +) +SELECT + "KBPO#" AS "kbpo#", + "KBITM#" AS "kbitm#", + KBQTYO AS kbqtyo, + KBQTYR AS kbqtyr, + KBQTYA AS kbqtya, + KBQTYJ AS kbqtyj, + RTRIM(KBOUNT) AS kbount, + RTRIM("KBPT#") AS "kbpt#", + RTRIM("KBVPT#") AS "kbvpt#", + KBUPRC AS kbuprc, + RTRIM(KBPUNT) AS kbpunt, + CASE WHEN KBRDAT IN (DATE('0001-01-01'), DATE('9999-12-31')) THEN NULL ELSE KBRDAT END AS kbrdat, + CASE WHEN KBCDAT IN (DATE('0001-01-01'), DATE('9999-12-31')) THEN NULL ELSE KBCDAT END AS kbcdat, + RTRIM(KBREQR) AS kbreqr, + "KBREQ#" AS "kbreq#", + "KBGL#" AS "kbgl#", + RTRIM("KBJOB#") AS "kbjob#", + "KBSEQ#" AS "kbseq#", + RTRIM("KBPRJ#") AS "kbprj#", + RTRIM(KBDEPT) AS kbdept, + RTRIM(KBCGST) AS kbcgst, + RTRIM(KBCPST) AS kbcpst, + RTRIM(KBISTS) AS kbists, + RTRIM(KBITYP) AS kbityp, + RTRIM(KBISTR) AS kbistr, + RTRIM(KBOSTR) AS kbostr, + KBEXTN AS kbextn, + KBGSTA AS kbgsta, + KBPSTA AS kbpsta, + RTRIM(KBPRGR) AS kbprgr, + RTRIM(KBCRCM) AS kbcrcm, + RTRIM(KBOTYP) AS kbotyp, + RTRIM(KBTOS) AS kbtos, + RTRIM(KBTAXG) AS kbtaxg, + RTRIM(KBTAXR) AS kbtaxr, + RTRIM(KBDPRC) AS kbdprc, + RTRIM(KBPLNT) AS kbplnt, + RTRIM(KBFUT1) AS kbfut1, + RTRIM(KBFUT2) AS kbfut2, + RTRIM(KBFUT3) AS kbfut3, + RTRIM(KBFUT4) AS kbfut4, + RTRIM(KBFUT5) AS kbfut5, + RTRIM(KBFUT6) AS kbfut6, + RTRIM(KBFUT7) AS kbfut7, + RTRIM(KBFUT8) AS kbfut8, + RTRIM(KBFUT9) AS kbfut9, + RTRIM(KBFUT10) AS kbfut10, + RTRIM(KBFUT11) AS kbfut11, + RTRIM(KBFUT12) AS kbfut12, + RTRIM(KBFUT13) AS kbfut13, + RTRIM(KBFUT14) AS kbfut14, + RTRIM(KBFUT15) AS kbfut15, + RTRIM(KBFUT16) AS kbfut16, + RTRIM(KBFUT17) AS kbfut17, + RTRIM(KBFUT18) AS kbfut18, + RTRIM(KBFUT19) AS kbfut19, + RTRIM(KBFUT20) AS kbfut20, + RTRIM(KBFUT21) AS kbfut21, + RTRIM(KBFUT22) AS kbfut22, + RTRIM(KBFUT23) AS kbfut23, + RTRIM(KBFUT24) AS kbfut24, + RTRIM(KBFUT25) AS kbfut25, + RTRIM(KBFUT26) AS kbfut26, + RTRIM(KBFUT27) AS kbfut27, + RTRIM(KBFUT28) AS kbfut28, + KBFUT29 AS kbfut29, + KBFUT30 AS kbfut30, + KBFUT31 AS kbfut31, + KBFUT32 AS kbfut32, + KBFUT33 AS kbfut33, + KBFUT34 AS kbfut34, + KBFUT35 AS kbfut35, + KBFUT36 AS kbfut36, + RTRIM(KBUSR1) AS kbusr1, + RTRIM(KBUSR2) AS kbusr2, + RTRIM(KBUSR3) AS kbusr3, + RTRIM(KBUSR4) AS kbusr4, + RTRIM(KBUSR5) AS kbusr5, + RTRIM(KBUSR6) AS kbusr6, + RTRIM(KBUSR7) AS kbusr7, + RTRIM(KBCORG) AS kbcorg, + RTRIM(KBROHD) AS kbrohd, + KBORDN AS kbordn, + KBOITM AS kboitm, + RTRIM(KBPPRT) AS kbpprt, + RTRIM(KBCSTC) AS kbcstc, + RTRIM(KBCSTT) AS kbcstt, + RTRIM(KBSRIN) AS kbsrin, + KBUNTFR AS kbuntfr, + KBSRGPC AS kbsrgpc, + KBFFRTC AS kbffrtc, + RTRIM(KBPGRP) AS kbpgrp +FROM LGDAT.POI i +INNER JOIN changed c + ON c.po = i."KBPO#" diff --git a/docs/cms_migration.md b/docs/cms_migration.md index 1f9256b..856be16 100644 --- a/docs/cms_migration.md +++ b/docs/cms_migration.md @@ -198,6 +198,61 @@ dates and clears the orphans while still skipping the 1953–66 junk), `pipekit icstt`, restore the `MAX(jhdate) - 7` resolver, then re-run the check above — it should read `IN SYNC`. Delete-by-date, so no `TRUNCATE` and no ACCESS EXCLUSIVE lock. +**Purchase order family (`poh` 138 / `poi` 139 / `poh_deleted` 144) — DONE +2026-08-18.** Converted from `full` (187k rows / 30 s and 529k / 75 s) to +`incremental`: **1,201 rows / 5.5 s** and **3,660 rows / 10 s**. `reconcile.py +--quick` reads IN SYNC on all 115 / 120 metrics, and a repeat run is a no-op. + +- **POH carries its own change stamp in its "future" fields** — + `KAFUT12` = *Date Created*, `KAFUT20` = *Date Updated*, both `CHAR(10)` ISO + `YYYY-MM-DD` (so watermark compares are lexical and blanks sort below any + floor). `KAFUT20` is populated on 113k of 187k rows — the gap is history that + predates the field, irrelevant to a forward incremental. Watermark + `poh_wm` / `poi_wm` = `MAX(kafut20)::date - 7` off `cms.poh`, capped at + `CURRENT_DATE`. +- Predicate: `KAFUT20 >= wm OR KAFUT12 >= wm OR KACRCM = '1'`. `KACRCM` + (1-Current / 2-Complete) leads most of POH's logicals, so the open hot set — + only **1,035 of 187k headers**, 2,374 of 529k lines — is a keyed read. +- **POI has no change stamp at all** (every `KBFUT*` is blank or a business + attribute), so it follows the header on a PO-grain join, plus 4 open lines + that hang off completed headers. `merge_key` is `kbpo#`, so the pull must stay + PO-grain — staging only the changed lines would drop the rest of the PO. +- **CMS logs this family too, and unlike `OCRIT` the header log is usable**: + `POHL` (4.6M rows, back to 2015) has logical **`POHLX1` keyed on `DQ0TMSP` + alone**; `POIL` has only `POILX1` keyed `DQ1PO#, DQ1TMSP`. Both were measured + as *redundant for change detection*: over 7 days POHL and the `KAFUT20/12` + window agree exactly (365 POs each, 0 either direction), and POIL reports 0 + POs POHL does not — a line edit always stamps the header. So no log join is + in the predicate. +- **`DQ0ACTN = '3'` is the delete action** (verified on PO 618973: last event is + an action 3, row gone from POH; 1/2 and 4/5 are before/after pairs). This is + the one thing the date fields structurally cannot see — an incremental merge + only deletes keys present in staging, so a purged PO would sit in the dest + forever, the same shape as the accepted `icstt` drift. Rate is ~**88 POs/yr** + (142,009 all-time since 2015, 140,924 currently absent from POH). +- `poh_deleted` (144) closes it: `DQ0ACTN = '3' AND DQ0TMSP >= '{wm}' AND NOT + EXISTS (… LGDAT.POH …)`, `incremental` on `dq0po#`, watermark + `MAX(SUBSTRING(dq0tmsp,1,10))::date - 7` off its own dest table. The + `NOT EXISTS` is what makes it exact instead of a guess about action ordering — + a re-used PO number is back in POH and drops out of the list on its own. Three + dest hooks, **order matters**: (1) `DELETE FROM cms.poh_deleted WHERE dq0po# + IN (SELECT kapo# FROM cms.poh)` clears the tombstone of any PO that is live + again *before* (2) deletes from `cms.poh` and (3) from `cms.poi` — without + hook 1, hook 2 could remove a row `poh` had just refreshed. First run + backfilled 140,924 rows in 13 s; steady state is a keyed range read of a + handful. Hooks need an explicit `connection_id` or the run fails with + `hook connection None not found`. +- **Line-level deletes need no POIL module.** Over 60 days, 66 POs had line + deletes: 36 still have a header and all 36 are already caught by `poi`'s + predicate (the PO-grain delete+reinsert removes the vanished line), and the + other 30 are whole-PO purges handled by `poh_deleted` keyed on the PO number. +- All three are in **no group** — manual runs for now. Whenever they are + scheduled, order must be `poh` → `poi` → `poh_deleted`: `poi`'s watermark + resolves off `cms.poh`, and `poh_deleted`'s hook 1 reads `cms.poh` expecting + it to be fresh. +- Note `WHERE TRUE` cannot be used in these extracts — DB2 for i has no boolean + literal and raises SQL0199. + ## Bucket 3 — special cases — decide, don't port as-is - **Reverse pushes PG→DB2** (`osmfs`, `reprice`, `reprice_test`): go Postgres→AS400,