pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
51ee71638f
pipekit/config/modules/poh.json
Paul Trowbridge d5618e9514 Sync purchase orders incrementally 
poh and poi were full truncate-reloads: 187k rows / 30 s and 529k / 75 s.
Both now pull only changed POs -- 1,201 rows / 5.5 s and 3,660 / 10 s --
and reconcile.py --quick reads IN SYNC on all 115 / 120 metrics.

POH turns out to carry its own change stamp in its "future" fields:
KAFUT12 is Date Created and KAFUT20 is Date Updated, both CHAR(10) ISO
text. Predicate is those two against the watermark, plus the open hot set
KACRCM = '1' -- 1,035 of 187k headers, and KACRCM leads most of POH's
logicals, so it is a keyed read. POI has no change stamp of its own, so
it joins the header's changed set at PO grain; its merge key is kbpo#, so
staging only the changed lines would drop the rest of the PO.

CMS logs this family (POHL, POIL), but both are redundant for change
detection: over 7 days POHL and the KAFUT20/12 window agree exactly, and
POIL reports no PO that POHL does not, so a line edit always stamps the
header. The logs' one unique contribution is deletes, which an
incremental merge structurally cannot see -- delete-by-key only touches
staged keys, so a purged PO would sit in the dest forever (~88 POs/yr).

poh_deleted closes that: DQ0ACTN = '3' is the delete action, and the
NOT EXISTS against live POH makes the list exact rather than a guess
about action ordering, since a re-used PO number is back in POH and drops
out on its own. Three dest hooks, in order -- clearing the tombstone of
any PO that is live again must happen before the deletes, or the second
hook could remove a row poh had just refreshed.

Whenever these are scheduled, run order must be poh -> poi ->
poh_deleted: poi's watermark resolves off cms.poh, and poh_deleted's
first hook reads cms.poh expecting it to be fresh.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-18 22:44:24 -04:00

695 lines
17 KiB
JSON
Raw Blame History

{
"name": "poh",
"source_connection": "S78030956",
"dest_connection": "usmidsap02",
"dest_table": "cms.poh",
"staging_table": "pipekit_staging.poh",
"merge_strategy": "incremental",
"merge_key": "kapo#",
"enabled": 1,
"dest_description": "Purchase Order Header 6.1",
"columns": [
{
"source_name": "KAPO#",
"source_type": "NUMERIC(9,0)",
"dest_name": "kapo#",
"dest_type": "numeric(9,0)",
"description": "Purchase Order Number"
},
{
"source_name": "KAOTYP",
"source_type": "CHAR(1)",
"dest_name": "kaotyp",
"dest_type": "text",
"description": "S-Standard, B-Blanket"
},
{
"source_name": "KARELD",
"source_type": "CHAR(1)",
"dest_name": "kareld",
"dest_type": "text",
"description": "Released"
},
{
"source_name": "KAODAT",
"source_type": "DATE",
"dest_name": "kaodat",
"dest_type": "date",
"description": "Order Date"
},
{
"source_name": "KAOVND",
"source_type": "CHAR(10)",
"dest_name": "kaovnd",
"dest_type": "text",
"description": "Vendor Code"
},
{
"source_name": "KAOVNM",
"source_type": "CHAR(30)",
"dest_name": "kaovnm",
"dest_type": "text",
"description": "Vendor Name"
},
{
"source_name": "KABVND",
"source_type": "CHAR(10)",
"dest_name": "kabvnd",
"dest_type": "text",
"description": "RemitTo Vendor Number"
},
{
"source_name": "KASVND",
"source_type": "CHAR(10)",
"dest_name": "kasvnd",
"dest_type": "text",
"description": "Shipto Vendor Number"
},
{
"source_name": "KASVNM",
"source_type": "CHAR(30)",
"dest_name": "kasvnm",
"dest_type": "text",
"description": "Shipto Vendor Name"
},
{
"source_name": "KASAD1",
"source_type": "CHAR(35)",
"dest_name": "kasad1",
"dest_type": "text",
"description": "Shipto Vendor Address1"
},
{
"source_name": "KASAD2",
"source_type": "CHAR(35)",
"dest_name": "kasad2",
"dest_type": "text",
"description": "Shipto Vendor Address2"
},
{
"source_name": "KASAD3",
"source_type": "CHAR(35)",
"dest_name": "kasad3",
"dest_type": "text",
"description": "Shipto Vendor Address3"
},
{
"source_name": "KASAD4",
"source_type": "CHAR(35)",
"dest_name": "kasad4",
"dest_type": "text",
"description": "Shipto Vendor Address4"
},
{
"source_name": "KASAD5",
"source_type": "CHAR(35)",
"dest_name": "kasad5",
"dest_type": "text",
"description": "Shipto Vendor Address5"
},
{
"source_name": "KASAD6",
"source_type": "CHAR(35)",
"dest_name": "kasad6",
"dest_type": "text",
"description": "Shipto Vendor Address6"
},
{
"source_name": "KASAD7",
"source_type": "CHAR(35)",
"dest_name": "kasad7",
"dest_type": "text",
"description": "Shipto Vendor Address7"
},
{
"source_name": "KASAD8",
"source_type": "CHAR(35)",
"dest_name": "kasad8",
"dest_type": "text",
"description": "Shipto Vendor Address8"
},
{
"source_name": "KASAD9",
"source_type": "CHAR(35)",
"dest_name": "kasad9",
"dest_type": "text",
"description": "Shipto Vendor Address9"
},
{
"source_name": "KASAD10",
"source_type": "CHAR(35)",
"dest_name": "kasad10",
"dest_type": "text",
"description": "Shipto Vendor Addres10"
},
{
"source_name": "KASVPC",
"source_type": "CHAR(10)",
"dest_name": "kasvpc",
"dest_type": "text",
"description": "Shipto Postal Code"
},
{
"source_name": "KACNTC",
"source_type": "CHAR(20)",
"dest_name": "kacntc",
"dest_type": "text",
"description": "Contact Person"
},
{
"source_name": "KAPHON",
"source_type": "CHAR(20)",
"dest_name": "kaphon",
"dest_type": "text",
"description": "Phone Number"
},
{
"source_name": "KATRMC",
"source_type": "CHAR(2)",
"dest_name": "katrmc",
"dest_type": "text",
"description": "Terms Code"
},
{
"source_name": "KASVIA",
"source_type": "CHAR(30)",
"dest_name": "kasvia",
"dest_type": "text",
"description": "Ship Via"
},
{
"source_name": "KAPCOL",
"source_type": "CHAR(1)",
"dest_name": "kapcol",
"dest_type": "text",
"description": "P-Prepaid,C-Collect,I-Charge"
},
{
"source_name": "KACONF",
"source_type": "CHAR(1)",
"dest_name": "kaconf",
"dest_type": "text",
"description": "Confirmation"
},
{
"source_name": "KAFOBC",
"source_type": "CHAR(3)",
"dest_name": "kafobc",
"dest_type": "text",
"description": "FOB Code"
},
{
"source_name": "KAOSTS",
"source_type": "CHAR(1)",
"dest_name": "kaosts",
"dest_type": "text",
"description": "Status - N-New,P-Printed,B-Backordered,C-Complete"
},
{
"source_name": "KAPSTS",
"source_type": "CHAR(1)",
"dest_name": "kapsts",
"dest_type": "text",
"description": "Print Status"
},
{
"source_name": "KAOGST",
"source_type": "DECIMAL(11,2)",
"dest_name": "kaogst",
"dest_type": "numeric(11,2)",
"description": "Recoverable Tax Amount"
},
{
"source_name": "KAOPST",
"source_type": "DECIMAL(11,2)",
"dest_name": "kaopst",
"dest_type": "numeric(11,2)",
"description": "Non-Recov Tax Amount"
},
{
"source_name": "KAOTOT",
"source_type": "DECIMAL(11,2)",
"dest_name": "kaotot",
"dest_type": "numeric(11,2)",
"description": "Order Total"
},
{
"source_name": "KABVAL",
"source_type": "DECIMAL(11,2)",
"dest_name": "kabval",
"dest_type": "numeric(11,2)",
"description": "Blanket Value"
},
{
"source_name": "KACURR",
"source_type": "CHAR(2)",
"dest_name": "kacurr",
"dest_type": "text",
"description": "Currency Code"
},
{
"source_name": "KABUYR",
"source_type": "CHAR(10)",
"dest_name": "kabuyr",
"dest_type": "text",
"description": "Buyer Code"
},
{
"source_name": "KALRL#",
"source_type": "NUMERIC(4,0)",
"dest_name": "kalrl#",
"dest_type": "numeric(4,0)",
"description": "Last Release Number"
},
{
"source_name": "KAQNUM",
"source_type": "NUMERIC(9,0)",
"dest_name": "kaqnum",
"dest_type": "numeric(9,0)",
"description": "P/O Quote Number"
},
{
"source_name": "KACRCM",
"source_type": "CHAR(1)",
"dest_name": "kacrcm",
"dest_type": "text",
"description": "1-Current,2-Complete"
},
{
"source_name": "KACARC",
"source_type": "CHAR(17)",
"dest_name": "kacarc",
"dest_type": "text",
"description": "Carrier Code"
},
{
"source_name": "KAFRAM",
"source_type": "DECIMAL(11,2)",
"dest_name": "kafram",
"dest_type": "numeric(11,2)",
"description": "Freight Amount"
},
{
"source_name": "KACBRC",
"source_type": "CHAR(10)",
"dest_name": "kacbrc",
"dest_type": "text",
"description": "Customs Broker Code"
},
{
"source_name": "KADUAM",
"source_type": "DECIMAL(11,2)",
"dest_name": "kaduam",
"dest_type": "numeric(11,2)",
"description": "Duty & Brokerage Fee"
},
{
"source_name": "KAHREA",
"source_type": "CHAR(1)",
"dest_name": "kahrea",
"dest_type": "text",
"description": "Hold Reason Code"
},
{
"source_name": "KAFAX",
"source_type": "CHAR(20)",
"dest_name": "kafax",
"dest_type": "text",
"description": "Fax Number"
},
{
"source_name": "KAIUSE",
"source_type": "CHAR(1)",
"dest_name": "kaiuse",
"dest_type": "text",
"description": "IN USE FLAG"
},
{
"source_name": "KAPURP",
"source_type": "CHAR(1)",
"dest_name": "kapurp",
"dest_type": "text",
"description": "Tran Purpse Code"
},
{
"source_name": "KAESTS",
"source_type": "CHAR(1)",
"dest_name": "kaests",
"dest_type": "text",
"description": "EDI Send Status"
},
{
"source_name": "KAASTS",
"source_type": "CHAR(1)",
"dest_name": "kaasts",
"dest_type": "text",
"description": "EDI Acknowledge Status"
},
{
"source_name": "KAPLNT",
"source_type": "CHAR(3)",
"dest_name": "kaplnt",
"dest_type": "text",
"description": "Plant Code"
},
{
"source_name": "KASCTY",
"source_type": "CHAR(30)",
"dest_name": "kascty",
"dest_type": "text",
"description": "Shipto City"
},
{
"source_name": "KASPOV",
"source_type": "CHAR(3)",
"dest_name": "kaspov",
"dest_type": "text",
"description": "Shipto State/Province"
},
{
"source_name": "KASCRY",
"source_type": "CHAR(3)",
"dest_name": "kascry",
"dest_type": "text",
"description": "Shipto Country"
},
{
"source_name": "KAFUT1",
"source_type": "CHAR(1)",
"dest_name": "kafut1",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "KAFUT2",
"source_type": "CHAR(1)",
"dest_name": "kafut2",
"dest_type": "text",
"description": "Ever Maintain"
},
{
"source_name": "KAFUT3",
"source_type": "CHAR(1)",
"dest_name": "kafut3",
"dest_type": "text",
"description": "Web Service In-Use"
},
{
"source_name": "KAFUT4",
"source_type": "CHAR(1)",
"dest_name": "kafut4",
"dest_type": "text",
"description": "FUTURE 4"
},
{
"source_name": "KAFUT5",
"source_type": "CHAR(1)",
"dest_name": "kafut5",
"dest_type": "text",
"description": "FUTURE 5"
},
{
"source_name": "KAFUT6",
"source_type": "CHAR(1)",
"dest_name": "kafut6",
"dest_type": "text",
"description": "FUTURE 6"
},
{
"source_name": "KAFUT7",
"source_type": "CHAR(1)",
"dest_name": "kafut7",
"dest_type": "text",
"description": "FUTURE 7"
},
{
"source_name": "KAFUT8",
"source_type": "CHAR(1)",
"dest_name": "kafut8",
"dest_type": "text",
"description": "FUTURE 8"
},
{
"source_name": "KAFUT9",
"source_type": "CHAR(1)",
"dest_name": "kafut9",
"dest_type": "text",
"description": "FUTURE 9"
},
{
"source_name": "KAFUT10",
"source_type": "CHAR(1)",
"dest_name": "kafut10",
"dest_type": "text",
"description": "FUTURE 10"
},
{
"source_name": "KAFUT11",
"source_type": "CHAR(10)",
"dest_name": "kafut11",
"dest_type": "text",
"description": "Created By User"
},
{
"source_name": "KAFUT12",
"source_type": "CHAR(10)",
"dest_name": "kafut12",
"dest_type": "text",
"description": "Date Created"
},
{
"source_name": "KAFUT13",
"source_type": "CHAR(10)",
"dest_name": "kafut13",
"dest_type": "text",
"description": "Time Created"
},
{
"source_name": "KAFUT14",
"source_type": "CHAR(10)",
"dest_name": "kafut14",
"dest_type": "text",
"description": "FUTURE 14"
},
{
"source_name": "KAFUT15",
"source_type": "CHAR(10)",
"dest_name": "kafut15",
"dest_type": "text",
"description": "FUTURE 15"
},
{
"source_name": "KAFUT16",
"source_type": "CHAR(10)",
"dest_name": "kafut16",
"dest_type": "text",
"description": "Last Aprvl/Reject Date"
},
{
"source_name": "KAFUT17",
"source_type": "CHAR(10)",
"dest_name": "kafut17",
"dest_type": "text",
"description": "Last Aprvl/Reject Time"
},
{
"source_name": "KAFUT18",
"source_type": "CHAR(10)",
"dest_name": "kafut18",
"dest_type": "text",
"description": "FUTURE 18"
},
{
"source_name": "KAFUT19",
"source_type": "CHAR(10)",
"dest_name": "kafut19",
"dest_type": "text",
"description": "Last Approval Checking Date"
},
{
"source_name": "KAFUT20",
"source_type": "CHAR(10)",
"dest_name": "kafut20",
"dest_type": "text",
"description": "Date Updated"
},
{
"source_name": "KAFUT21",
"source_type": "CHAR(20)",
"dest_name": "kafut21",
"dest_type": "text",
"description": "Time Updated"
},
{
"source_name": "KAFUT22",
"source_type": "CHAR(20)",
"dest_name": "kafut22",
"dest_type": "text",
"description": "Last Approval Checking Time"
},
{
"source_name": "KAFUT23",
"source_type": "CHAR(20)",
"dest_name": "kafut23",
"dest_type": "text",
"description": "FUTURE 23"
},
{
"source_name": "KAFUT24",
"source_type": "CHAR(20)",
"dest_name": "kafut24",
"dest_type": "text",
"description": "FUTURE 24"
},
{
"source_name": "KAFUT25",
"source_type": "CHAR(20)",
"dest_name": "kafut25",
"dest_type": "text",
"description": "FUTURE 25"
},
{
"source_name": "KAFUT26",
"source_type": "CHAR(21)",
"dest_name": "kafut26",
"dest_type": "text",
"description": "FUTURE 26"
},
{
"source_name": "KAFUT27",
"source_type": "CHAR(22)",
"dest_name": "kafut27",
"dest_type": "text",
"description": "FUTURE 27"
},
{
"source_name": "KAFUT28",
"source_type": "CHAR(23)",
"dest_name": "kafut28",
"dest_type": "text",
"description": "FUTURE 28"
},
{
"source_name": "KAFUT29",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut29",
"dest_type": "numeric(15,5)",
"description": "FUTURE 29"
},
{
"source_name": "KAFUT30",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut30",
"dest_type": "numeric(15,5)",
"description": "FUTURE 30"
},
{
"source_name": "KAFUT31",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut31",
"dest_type": "numeric(15,5)",
"description": "FUTURE 31"
},
{
"source_name": "KAFUT32",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut32",
"dest_type": "numeric(15,5)",
"description": "FUTURE 32"
},
{
"source_name": "KAFUT33",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut33",
"dest_type": "numeric(15,5)",
"description": "FUTURE 33"
},
{
"source_name": "KAFUT34",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut34",
"dest_type": "numeric(15,5)",
"description": "FUTURE 34"
},
{
"source_name": "KAFUT35",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut35",
"dest_type": "numeric(15,5)",
"description": "FUTURE 35"
},
{
"source_name": "KAFUT36",
"source_type": "NUMERIC(15,5)",
"dest_name": "kafut36",
"dest_type": "numeric(15,5)",
"description": "FUTURE 36"
},
{
"source_name": "KAUSR1",
"source_type": "CHAR(10)",
"dest_name": "kausr1",
"dest_type": "text",
"description": "User 1"
},
{
"source_name": "KAUSR2",
"source_type": "CHAR(10)",
"dest_name": "kausr2",
"dest_type": "text",
"description": "User 2"
},
{
"source_name": "KAUSR3",
"source_type": "CHAR(10)",
"dest_name": "kausr3",
"dest_type": "text",
"description": "User 3"
},
{
"source_name": "KAUSR4",
"source_type": "CHAR(10)",
"dest_name": "kausr4",
"dest_type": "text",
"description": "User 4"
},
{
"source_name": "KAUSR5",
"source_type": "CHAR(10)",
"dest_name": "kausr5",
"dest_type": "text",
"description": "User 5"
},
{
"source_name": "KAUSR6",
"source_type": "CHAR(10)",
"dest_name": "kausr6",
"dest_type": "text",
"description": "User 6"
},
{
"source_name": "KAUSR7",
"source_type": "CHAR(10)",
"dest_name": "kausr7",
"dest_type": "text",
"description": "User 7"
},
{
"source_name": "KAEDIRS",
"source_type": "CHAR(1)",
"dest_name": "kaedirs",
"dest_type": "text",
"description": "EDI Re-send"
}
],
"watermarks": [
{
"name": "poh_wm",
"connection": "usmidsap02",
"resolver_sql": "SELECT TO_CHAR(LEAST(COALESCE(MAX(NULLIF(kafut20, ''))::DATE, DATE '1900-01-01'), CURRENT_DATE) - 7, 'YYYY-MM-DD') FROM cms.poh",
"default_value": null
}
],
"hooks": []
}
Reference in New Issue View Git Blame Copy Permalink