Paul Trowbridge
b855ab91d7
The Accounts Payable group ran ~9.5 min, 83% of it in four full reloads. The cost is the GPSERVER linked-server hop, not row volume (pm30200 managed only ~590 rows/sec), so cutting rows crossing the link is the whole win. Predicates go inside OPENQUERY to push down to the remote — a 4-part name would drag the table across before filtering. pm30200 own DEX_ROW_TS pm30600 changed set: parent PM30200 pm30300 changed set: PM30200 union PM20000 All three key on (vchrnmbr, doctype), verified exactly unique in PM30200 at 108,803/108,803. Watermark resolves off gp.pm30200 with a 7-day lookback, so run order within the group must stay pm30200 first and the lookback must exceed the sync interval. The two-parent union on pm30300 is load-bearing: 38 rows have a parent only in the open table, and a PM30200-only join would strand them permanently. pm30600 needs no union (0 orphans, verified). pm30700 stays full — 213 of its rows have no parent in either table, so no changed-set join can reach them, and it only costs 13s. pm20100 and pm00400 stay full because open and index tables delete rows, which a delete-by-key incremental structurally cannot propagate (cf. icstt). 391s -> 15s. reconcile.py --super-quick reports IN SYNC for all three. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
570 lines
14 KiB
JSON
570 lines
14 KiB
JSON
{
|
|
"name": "pm30200",
|
|
"source_connection": "sqlserver",
|
|
"dest_connection": "usmidsap02",
|
|
"dest_table": "gp.pm30200",
|
|
"staging_table": "pipekit_staging.pm30200",
|
|
"merge_strategy": "incremental",
|
|
"merge_key": "vchrnmbr, doctype",
|
|
"enabled": 1,
|
|
"dest_description": "Dynamics GP payables history",
|
|
"columns": [
|
|
{
|
|
"source_name": "VCHRNMBR",
|
|
"source_type": "CHAR(21)",
|
|
"dest_name": "vchrnmbr",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VENDORID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "vendorid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DOCTYPE",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "doctype",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DOCDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "docdate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DOCNUMBR",
|
|
"source_type": "CHAR(21)",
|
|
"dest_name": "docnumbr",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DOCAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "docamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CURTRXAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "curtrxam",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DISTKNAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "distknam",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DISCAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "discamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DSCDLRAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "dscdlram",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "BACHNUMB",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "bachnumb",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TRXSORCE",
|
|
"source_type": "CHAR(13)",
|
|
"dest_name": "trxsorce",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "BCHSOURC",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "bchsourc",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DISCDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "discdate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DUEDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "duedate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PORDNMBR",
|
|
"source_type": "CHAR(21)",
|
|
"dest_name": "pordnmbr",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TEN99AMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "ten99amnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "WROFAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "wrofamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DISAMTAV",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "disamtav",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TRXDSCRN",
|
|
"source_type": "CHAR(31)",
|
|
"dest_name": "trxdscrn",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "UN1099AM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "un1099am",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "BKTPURAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "bktpuram",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "BKTFRTAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "bktfrtam",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "BKTMSCAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "bktmscam",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VOIDED",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "voided",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "HOLD",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "hold",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CHEKBKID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "chekbkid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DINVPDOF",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "dinvpdof",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PPSAMDED",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "ppsamded",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PPSTAXRT",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "ppstaxrt",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PGRAMSBJ",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "pgramsbj",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "GSTDSAMT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "gstdsamt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "POSTEDDT",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "posteddt",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PTDUSRID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "ptdusrid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "MODIFDT",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "modifdt",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "MDFUSRID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "mdfusrid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PYENTTYP",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "pyenttyp",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CARDNAME",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "cardname",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PRCHAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "prchamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TRDISAMT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "trdisamt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "MSCCHAMT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "mscchamt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "FRTAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "frtamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TAXAMNT",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "taxamnt",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TTLPYMTS",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "ttlpymts",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CURNCYID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "curncyid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PYMTRMID",
|
|
"source_type": "CHAR(21)",
|
|
"dest_name": "pymtrmid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "SHIPMTHD",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "shipmthd",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TAXSCHID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "taxschid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PCHSCHID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "pchschid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "FRTSCHID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "frtschid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "MSCSCHID",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "mscschid",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PSTGDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "pstgdate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DISAVTKN",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "disavtkn",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CNTRLTYP",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "cntrltyp",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "NOTEINDX",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "noteindx",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PRCTDISC",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "prctdisc",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "RETNAGAM",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "retnagam",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VOIDPDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "voidpdate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "ICTRX",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "ictrx",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "Tax_Date",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "tax_date",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PRCHDATE",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "prchdate",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CORRCTN",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "corrctn",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "SIMPLIFD",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "simplifd",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "APLYWITH",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "aplywith",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "Electronic",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "electronic",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "ECTRX",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "ectrx",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DocPrinted",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "docprinted",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TaxInvReqd",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "taxinvreqd",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VNDCHKNM",
|
|
"source_type": "CHAR(65)",
|
|
"dest_name": "vndchknm",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "BackoutTradeDisc",
|
|
"source_type": "NUMERIC(19,5)",
|
|
"dest_name": "backouttradedisc",
|
|
"dest_type": "numeric(19,5)",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "CBVAT",
|
|
"source_type": "TINYINT",
|
|
"dest_name": "cbvat",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "VADCDTRO",
|
|
"source_type": "CHAR(15)",
|
|
"dest_name": "vadcdtro",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TEN99TYPE",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "ten99type",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "TEN99BOXNUMBER",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "ten99boxnumber",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "PONUMBER",
|
|
"source_type": "CHAR(17)",
|
|
"dest_name": "ponumber",
|
|
"dest_type": "text",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "Workflow_Status",
|
|
"source_type": "SMALLINT",
|
|
"dest_name": "workflow_status",
|
|
"dest_type": "smallint",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DEX_ROW_TS",
|
|
"source_type": "DATETIME",
|
|
"dest_name": "dex_row_ts",
|
|
"dest_type": "timestamp",
|
|
"description": null
|
|
},
|
|
{
|
|
"source_name": "DEX_ROW_ID",
|
|
"source_type": "INT",
|
|
"dest_name": "dex_row_id",
|
|
"dest_type": "integer",
|
|
"description": null
|
|
}
|
|
],
|
|
"watermarks": [
|
|
{
|
|
"name": "pm30200_ts",
|
|
"connection": "usmidsap02",
|
|
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
|
|
"default_value": "1900-01-01"
|
|
}
|
|
],
|
|
"hooks": [],
|
|
"description": "Dynamics GP payables history"
|
|
}
|