pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
7de9d83589
pipekit/config/modules/poi.json
Paul Trowbridge d5618e9514 Sync purchase orders incrementally 
poh and poi were full truncate-reloads: 187k rows / 30 s and 529k / 75 s.
Both now pull only changed POs -- 1,201 rows / 5.5 s and 3,660 / 10 s --
and reconcile.py --quick reads IN SYNC on all 115 / 120 metrics.

POH turns out to carry its own change stamp in its "future" fields:
KAFUT12 is Date Created and KAFUT20 is Date Updated, both CHAR(10) ISO
text. Predicate is those two against the watermark, plus the open hot set
KACRCM = '1' -- 1,035 of 187k headers, and KACRCM leads most of POH's
logicals, so it is a keyed read. POI has no change stamp of its own, so
it joins the header's changed set at PO grain; its merge key is kbpo#, so
staging only the changed lines would drop the rest of the PO.

CMS logs this family (POHL, POIL), but both are redundant for change
detection: over 7 days POHL and the KAFUT20/12 window agree exactly, and
POIL reports no PO that POHL does not, so a line edit always stamps the
header. The logs' one unique contribution is deletes, which an
incremental merge structurally cannot see -- delete-by-key only touches
staged keys, so a purged PO would sit in the dest forever (~88 POs/yr).

poh_deleted closes that: DQ0ACTN = '3' is the delete action, and the
NOT EXISTS against live POH makes the list exact rather than a guess
about action ordering, since a re-used PO number is back in POH and drops
out on its own. Three dest hooks, in order -- clearing the tombstone of
any PO that is live again must happen before the deletes, or the second
hook could remove a row poh had just refreshed.

Whenever these are scheduled, run order must be poh -> poi ->
poh_deleted: poi's watermark resolves off cms.poh, and poh_deleted's
first hook reads cms.poh expecting it to be fresh.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-18 22:44:24 -04:00

667 lines
16 KiB
JSON
Raw Blame History

{
"name": "poi",
"source_connection": "S78030956",
"dest_connection": "usmidsap02",
"dest_table": "cms.poi",
"staging_table": "pipekit_staging.poi",
"merge_strategy": "incremental",
"merge_key": "kbpo#",
"enabled": 1,
"dest_description": "Purchase Order Items 6.0",
"columns": [
{
"source_name": "KBPO#",
"source_type": "NUMERIC(9,0)",
"dest_name": "kbpo#",
"dest_type": "numeric(9,0)",
"description": "Purchase Order Number"
},
{
"source_name": "KBITM#",
"source_type": "NUMERIC(3,0)",
"dest_name": "kbitm#",
"dest_type": "numeric(3,0)",
"description": "P/O Item #"
},
{
"source_name": "KBQTYO",
"source_type": "DECIMAL(11,2)",
"dest_name": "kbqtyo",
"dest_type": "numeric(11,2)",
"description": "Quantity Ordered"
},
{
"source_name": "KBQTYR",
"source_type": "DECIMAL(15,5)",
"dest_name": "kbqtyr",
"dest_type": "numeric(15,5)",
"description": "Quantity Received"
},
{
"source_name": "KBQTYA",
"source_type": "DECIMAL(15,5)",
"dest_name": "kbqtya",
"dest_type": "numeric(15,5)",
"description": "Quantity Approved"
},
{
"source_name": "KBQTYJ",
"source_type": "DECIMAL(15,5)",
"dest_name": "kbqtyj",
"dest_type": "numeric(15,5)",
"description": "Quantity Rejected"
},
{
"source_name": "KBOUNT",
"source_type": "CHAR(3)",
"dest_name": "kbount",
"dest_type": "text",
"description": "Order Unit"
},
{
"source_name": "KBPT#",
"source_type": "CHAR(20)",
"dest_name": "kbpt#",
"dest_type": "text",
"description": "Part Number"
},
{
"source_name": "KBVPT#",
"source_type": "CHAR(30)",
"dest_name": "kbvpt#",
"dest_type": "text",
"description": "Vendor Part Number"
},
{
"source_name": "KBUPRC",
"source_type": "DECIMAL(13,5)",
"dest_name": "kbuprc",
"dest_type": "numeric(13,5)",
"description": "Unit Price"
},
{
"source_name": "KBPUNT",
"source_type": "CHAR(3)",
"dest_name": "kbpunt",
"dest_type": "text",
"description": "Price Unit"
},
{
"source_name": "KBRDAT",
"source_type": "DATE",
"dest_name": "kbrdat",
"dest_type": "date",
"description": "Date Required"
},
{
"source_name": "KBCDAT",
"source_type": "DATE",
"dest_name": "kbcdat",
"dest_type": "date",
"description": "Date Confirmed"
},
{
"source_name": "KBREQR",
"source_type": "CHAR(10)",
"dest_name": "kbreqr",
"dest_type": "text",
"description": "Requisitioner"
},
{
"source_name": "KBREQ#",
"source_type": "NUMERIC(9,0)",
"dest_name": "kbreq#",
"dest_type": "numeric(9,0)",
"description": "P/O Requisition Number"
},
{
"source_name": "KBGL#",
"source_type": "NUMERIC(10,0)",
"dest_name": "kbgl#",
"dest_type": "numeric(10,0)",
"description": "G/L Account Number"
},
{
"source_name": "KBJOB#",
"source_type": "CHAR(10)",
"dest_name": "kbjob#",
"dest_type": "text",
"description": "Job Number"
},
{
"source_name": "KBSEQ#",
"source_type": "NUMERIC(3,0)",
"dest_name": "kbseq#",
"dest_type": "numeric(3,0)",
"description": "Job Sequence Number"
},
{
"source_name": "KBPRJ#",
"source_type": "CHAR(20)",
"dest_name": "kbprj#",
"dest_type": "text",
"description": "Project Number"
},
{
"source_name": "KBDEPT",
"source_type": "CHAR(2)",
"dest_name": "kbdept",
"dest_type": "text",
"description": "Department Number"
},
{
"source_name": "KBCGST",
"source_type": "CHAR(1)",
"dest_name": "kbcgst",
"dest_type": "text",
"description": "OBSOLETE - DO NOT USE"
},
{
"source_name": "KBCPST",
"source_type": "CHAR(1)",
"dest_name": "kbcpst",
"dest_type": "text",
"description": "OBSOLETE - DO NOT USE"
},
{
"source_name": "KBISTS",
"source_type": "CHAR(1)",
"dest_name": "kbists",
"dest_type": "text",
"description": "Status - O-Open,B-Backordered,C-Complete"
},
{
"source_name": "KBITYP",
"source_type": "CHAR(1)",
"dest_name": "kbityp",
"dest_type": "text",
"description": "Type - -Normal,D-Description,O-Outside Service,S-"
},
{
"source_name": "KBISTR",
"source_type": "CHAR(6)",
"dest_name": "kbistr",
"dest_type": "text",
"description": "Inventory Stockroom"
},
{
"source_name": "KBOSTR",
"source_type": "CHAR(3)",
"dest_name": "kbostr",
"dest_type": "text",
"description": "OBSOLETE FIELD"
},
{
"source_name": "KBEXTN",
"source_type": "DECIMAL(11,2)",
"dest_name": "kbextn",
"dest_type": "numeric(11,2)",
"description": "Total Including Taxes"
},
{
"source_name": "KBGSTA",
"source_type": "DECIMAL(11,2)",
"dest_name": "kbgsta",
"dest_type": "numeric(11,2)",
"description": "Recoverable Tax Amount"
},
{
"source_name": "KBPSTA",
"source_type": "DECIMAL(11,2)",
"dest_name": "kbpsta",
"dest_type": "numeric(11,2)",
"description": "Non-Recov Tax Amount"
},
{
"source_name": "KBPRGR",
"source_type": "CHAR(1)",
"dest_name": "kbprgr",
"dest_type": "text",
"description": "Progressive Release Y/N"
},
{
"source_name": "KBCRCM",
"source_type": "CHAR(1)",
"dest_name": "kbcrcm",
"dest_type": "text",
"description": "1-Current,2-Complete"
},
{
"source_name": "KBOTYP",
"source_type": "CHAR(1)",
"dest_name": "kbotyp",
"dest_type": "text",
"description": "S-Standard, B-blanket"
},
{
"source_name": "KBTOS",
"source_type": "CHAR(1)",
"dest_name": "kbtos",
"dest_type": "text",
"description": "Transferred Outside Service"
},
{
"source_name": "KBTAXG",
"source_type": "CHAR(3)",
"dest_name": "kbtaxg",
"dest_type": "text",
"description": "Tax Group Code"
},
{
"source_name": "KBTAXR",
"source_type": "CHAR(1)",
"dest_name": "kbtaxr",
"dest_type": "text",
"description": "Tax Rate Code"
},
{
"source_name": "KBDPRC",
"source_type": "CHAR(1)",
"dest_name": "kbdprc",
"dest_type": "text",
"description": "Dynamic Price Flag"
},
{
"source_name": "KBPLNT",
"source_type": "CHAR(3)",
"dest_name": "kbplnt",
"dest_type": "text",
"description": "Plant Code"
},
{
"source_name": "KBFUT1",
"source_type": "CHAR(1)",
"dest_name": "kbfut1",
"dest_type": "text",
"description": "FUTURE 1"
},
{
"source_name": "KBFUT2",
"source_type": "CHAR(1)",
"dest_name": "kbfut2",
"dest_type": "text",
"description": "FUTURE 2"
},
{
"source_name": "KBFUT3",
"source_type": "CHAR(1)",
"dest_name": "kbfut3",
"dest_type": "text",
"description": "FUTURE 3"
},
{
"source_name": "KBFUT4",
"source_type": "CHAR(1)",
"dest_name": "kbfut4",
"dest_type": "text",
"description": "FUTURE 4"
},
{
"source_name": "KBFUT5",
"source_type": "CHAR(1)",
"dest_name": "kbfut5",
"dest_type": "text",
"description": "FUTURE 5"
},
{
"source_name": "KBFUT6",
"source_type": "CHAR(1)",
"dest_name": "kbfut6",
"dest_type": "text",
"description": "FUTURE 6"
},
{
"source_name": "KBFUT7",
"source_type": "CHAR(1)",
"dest_name": "kbfut7",
"dest_type": "text",
"description": "FUTURE 7"
},
{
"source_name": "KBFUT8",
"source_type": "CHAR(1)",
"dest_name": "kbfut8",
"dest_type": "text",
"description": "Initialized Min. Shipment Completion %"
},
{
"source_name": "KBFUT9",
"source_type": "CHAR(1)",
"dest_name": "kbfut9",
"dest_type": "text",
"description": "FUTURE 9"
},
{
"source_name": "KBFUT10",
"source_type": "CHAR(1)",
"dest_name": "kbfut10",
"dest_type": "text",
"description": "Update Last Actual"
},
{
"source_name": "KBFUT11",
"source_type": "CHAR(10)",
"dest_name": "kbfut11",
"dest_type": "text",
"description": "FUTURE 11"
},
{
"source_name": "KBFUT12",
"source_type": "CHAR(10)",
"dest_name": "kbfut12",
"dest_type": "text",
"description": "Revision Level"
},
{
"source_name": "KBFUT13",
"source_type": "CHAR(10)",
"dest_name": "kbfut13",
"dest_type": "text",
"description": "FUTURE 13"
},
{
"source_name": "KBFUT14",
"source_type": "CHAR(10)",
"dest_name": "kbfut14",
"dest_type": "text",
"description": "FUTURE 14"
},
{
"source_name": "KBFUT15",
"source_type": "CHAR(10)",
"dest_name": "kbfut15",
"dest_type": "text",
"description": "FUTURE 15"
},
{
"source_name": "KBFUT16",
"source_type": "CHAR(10)",
"dest_name": "kbfut16",
"dest_type": "text",
"description": "FUTURE 16"
},
{
"source_name": "KBFUT17",
"source_type": "CHAR(10)",
"dest_name": "kbfut17",
"dest_type": "text",
"description": "FUTURE 17"
},
{
"source_name": "KBFUT18",
"source_type": "CHAR(10)",
"dest_name": "kbfut18",
"dest_type": "text",
"description": "FUTURE 18"
},
{
"source_name": "KBFUT19",
"source_type": "CHAR(10)",
"dest_name": "kbfut19",
"dest_type": "text",
"description": "FUTURE 19"
},
{
"source_name": "KBFUT20",
"source_type": "CHAR(10)",
"dest_name": "kbfut20",
"dest_type": "text",
"description": "FUTURE 20"
},
{
"source_name": "KBFUT21",
"source_type": "CHAR(20)",
"dest_name": "kbfut21",
"dest_type": "text",
"description": "FUTURE 21"
},
{
"source_name": "KBFUT22",
"source_type": "CHAR(20)",
"dest_name": "kbfut22",
"dest_type": "text",
"description": "FUTURE 22"
},
{
"source_name": "KBFUT23",
"source_type": "CHAR(20)",
"dest_name": "kbfut23",
"dest_type": "text",
"description": "FUTURE 23"
},
{
"source_name": "KBFUT24",
"source_type": "CHAR(20)",
"dest_name": "kbfut24",
"dest_type": "text",
"description": "FUTURE 24"
},
{
"source_name": "KBFUT25",
"source_type": "CHAR(20)",
"dest_name": "kbfut25",
"dest_type": "text",
"description": "FUTURE 25"
},
{
"source_name": "KBFUT26",
"source_type": "CHAR(21)",
"dest_name": "kbfut26",
"dest_type": "text",
"description": "FUTURE 26"
},
{
"source_name": "KBFUT27",
"source_type": "CHAR(22)",
"dest_name": "kbfut27",
"dest_type": "text",
"description": "FUTURE 27"
},
{
"source_name": "KBFUT28",
"source_type": "CHAR(23)",
"dest_name": "kbfut28",
"dest_type": "text",
"description": "FUTURE 28"
},
{
"source_name": "KBFUT29",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut29",
"dest_type": "numeric(15,5)",
"description": "Allowed Over-shipment Percent"
},
{
"source_name": "KBFUT30",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut30",
"dest_type": "numeric(15,5)",
"description": "Minimum Shipment Completion %"
},
{
"source_name": "KBFUT31",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut31",
"dest_type": "numeric(15,5)",
"description": "FUTURE 31"
},
{
"source_name": "KBFUT32",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut32",
"dest_type": "numeric(15,5)",
"description": "FUTURE 32"
},
{
"source_name": "KBFUT33",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut33",
"dest_type": "numeric(15,5)",
"description": "FUTURE 33"
},
{
"source_name": "KBFUT34",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut34",
"dest_type": "numeric(15,5)",
"description": "FUTURE 34"
},
{
"source_name": "KBFUT35",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut35",
"dest_type": "numeric(15,5)",
"description": "FUTURE 35"
},
{
"source_name": "KBFUT36",
"source_type": "NUMERIC(15,5)",
"dest_name": "kbfut36",
"dest_type": "numeric(15,5)",
"description": "FUTURE 36"
},
{
"source_name": "KBUSR1",
"source_type": "CHAR(10)",
"dest_name": "kbusr1",
"dest_type": "text",
"description": "User 1"
},
{
"source_name": "KBUSR2",
"source_type": "CHAR(10)",
"dest_name": "kbusr2",
"dest_type": "text",
"description": "User 2"
},
{
"source_name": "KBUSR3",
"source_type": "CHAR(10)",
"dest_name": "kbusr3",
"dest_type": "text",
"description": "User 3"
},
{
"source_name": "KBUSR4",
"source_type": "CHAR(10)",
"dest_name": "kbusr4",
"dest_type": "text",
"description": "User 4"
},
{
"source_name": "KBUSR5",
"source_type": "CHAR(10)",
"dest_name": "kbusr5",
"dest_type": "text",
"description": "User 5"
},
{
"source_name": "KBUSR6",
"source_type": "CHAR(10)",
"dest_name": "kbusr6",
"dest_type": "text",
"description": "User 6"
},
{
"source_name": "KBUSR7",
"source_type": "CHAR(10)",
"dest_name": "kbusr7",
"dest_type": "text",
"description": "User 7"
},
{
"source_name": "KBCORG",
"source_type": "CHAR(3)",
"dest_name": "kbcorg",
"dest_type": "text",
"description": "Country Of Origin (Code)"
},
{
"source_name": "KBROHD",
"source_type": "CHAR(1)",
"dest_name": "kbrohd",
"dest_type": "text",
"description": "Put Receipt on Hold For Voucher Creation"
},
{
"source_name": "KBORDN",
"source_type": "NUMERIC(9,0)",
"dest_name": "kbordn",
"dest_type": "numeric(9,0)",
"description": "Customer Order Number"
},
{
"source_name": "KBOITM",
"source_type": "NUMERIC(3,0)",
"dest_name": "kboitm",
"dest_type": "numeric(3,0)",
"description": "Ref. Sales Ord. Item#"
},
{
"source_name": "KBPPRT",
"source_type": "CHAR(20)",
"dest_name": "kbpprt",
"dest_type": "text",
"description": "Project Part Number"
},
{
"source_name": "KBCSTC",
"source_type": "CHAR(2)",
"dest_name": "kbcstc",
"dest_type": "text",
"description": "Project Cost Code"
},
{
"source_name": "KBCSTT",
"source_type": "CHAR(1)",
"dest_name": "kbcstt",
"dest_type": "text",
"description": "Project Cost Type"
},
{
"source_name": "KBSRIN",
"source_type": "CHAR(1)",
"dest_name": "kbsrin",
"dest_type": "text",
"description": "Surcharge Indicator"
},
{
"source_name": "KBUNTFR",
"source_type": "NUMERIC(11,2)",
"dest_name": "kbuntfr",
"dest_type": "numeric(11,2)",
"description": "Per-Unit Freight Cost"
},
{
"source_name": "KBSRGPC",
"source_type": "NUMERIC(8,5)",
"dest_name": "kbsrgpc",
"dest_type": "numeric(8,5)",
"description": "Surcharge Percentage"
},
{
"source_name": "KBFFRTC",
"source_type": "NUMERIC(11,2)",
"dest_name": "kbffrtc",
"dest_type": "numeric(11,2)",
"description": "Flat Freight Cost"
},
{
"source_name": "KBPGRP",
"source_type": "CHAR(20)",
"dest_name": "kbpgrp",
"dest_type": "text",
"description": "Commodity Category Code"
}
],
"watermarks": [
{
"name": "poi_wm",
"connection": "usmidsap02",
"resolver_sql": "SELECT TO_CHAR(LEAST(COALESCE(MAX(NULLIF(kafut20, ''))::DATE, DATE '1900-01-01'), CURRENT_DATE) - 7, 'YYYY-MM-DD') FROM cms.poh",
"default_value": null
}
],
"hooks": []
}
Reference in New Issue View Git Blame Copy Permalink