Paul Trowbridge
b855ab91d7
The Accounts Payable group ran ~9.5 min, 83% of it in four full reloads. The cost is the GPSERVER linked-server hop, not row volume (pm30200 managed only ~590 rows/sec), so cutting rows crossing the link is the whole win. Predicates go inside OPENQUERY to push down to the remote — a 4-part name would drag the table across before filtering. pm30200 own DEX_ROW_TS pm30600 changed set: parent PM30200 pm30300 changed set: PM30200 union PM20000 All three key on (vchrnmbr, doctype), verified exactly unique in PM30200 at 108,803/108,803. Watermark resolves off gp.pm30200 with a 7-day lookback, so run order within the group must stay pm30200 first and the lookback must exceed the sync interval. The two-parent union on pm30300 is load-bearing: 38 rows have a parent only in the open table, and a PM30200-only join would strand them permanently. pm30600 needs no union (0 orphans, verified). pm30700 stays full — 213 of its rows have no parent in either table, so no changed-set join can reach them, and it only costs 13s. pm20100 and pm00400 stay full because open and index tables delete rows, which a delete-by-key incremental structurally cannot propagate (cf. icstt). 391s -> 15s. reconcile.py --super-quick reports IN SYNC for all three. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
185 lines
4.4 KiB
JSON
185 lines
4.4 KiB
JSON
{
|
|
"name": "pm30600",
|
|
"source_connection": "sqlserver",
|
|
"dest_connection": "usmidsap02",
|
|
"dest_table": "gp.pm30600",
|
|
"staging_table": "pipekit_staging.pm30600",
|
|
"merge_strategy": "incremental",
|
|
"merge_key": "vchrnmbr, doctype",
|
|
"enabled": 1,
|
|
"dest_description": "Dynamics GP payables GL distribution history",
|
|
"columns": [
|
|
{
|
|
"source_name": "DOCTYPE",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "doctype",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VCHRNMBR",
|
|
"source_type": "CHAR(21)",
|
|
"dest_name": "vchrnmbr",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DSTSQNUM",
|
|
"source_type": "INT",
|
|
"dest_name": "dstsqnum",
|
|
"dest_type": "integer",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CNTRLTYP",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "cntrltyp",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CRDTAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "crdtamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DEBITAMT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "debitamt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DSTINDX",
|
|
"source_type": "INT",
|
|
"dest_name": "dstindx",
|
|
"dest_type": "integer",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DISTTYPE",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "disttype",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CHANGED",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "changed",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "USERID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "userid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PSTGSTUS",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "pstgstus",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VENDORID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "vendorid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TRXSORCE",
|
|
"source_type": "CHAR(13)",
|
|
"dest_name": "trxsorce",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PSTGDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "pstgdate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CURNCYID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "curncyid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CURRNIDX",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "currnidx",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "ORCRDAMT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "orcrdamt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "ORDBTAMT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "ordbtamt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "APTVCHNM",
|
|
"source_type": "CHAR(21)",
|
|
"dest_name": "aptvchnm",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "APTODCTY",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "aptodcty",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "SPCLDIST",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "spcldist",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DistRef",
|
|
"source_type": "CHAR(31)",
|
|
"dest_name": "distref",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DEX_ROW_ID",
|
|
"source_type": "INT",
|
|
"dest_name": "dex_row_id",
|
|
"dest_type": "integer",
|
|
"description": null
|
|
}
|
|
],
|
|
"watermarks": [
|
|
{
|
|
"name": "pm30600_ts",
|
|
"connection": "usmidsap02",
|
|
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
|
|
"default_value": "1900-01-01"
|
|
}
|
|
],
|
|
"hooks": [],
|
|
"description": "Dynamics GP payables GL distribution history"
|
|
}
|