pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
855dd1a402
pipekit/config/modules/pm30200.json
Paul Trowbridge b855ab91d7 Make the three AP history modules incremental 
The Accounts Payable group ran ~9.5 min, 83% of it in four full
reloads. The cost is the GPSERVER linked-server hop, not row volume
(pm30200 managed only ~590 rows/sec), so cutting rows crossing the link
is the whole win. Predicates go inside OPENQUERY to push down to the
remote — a 4-part name would drag the table across before filtering.

  pm30200  own DEX_ROW_TS
  pm30600  changed set: parent PM30200
  pm30300  changed set: PM30200 union PM20000

All three key on (vchrnmbr, doctype), verified exactly unique in
PM30200 at 108,803/108,803. Watermark resolves off gp.pm30200 with a
7-day lookback, so run order within the group must stay pm30200 first
and the lookback must exceed the sync interval.

The two-parent union on pm30300 is load-bearing: 38 rows have a parent
only in the open table, and a PM30200-only join would strand them
permanently. pm30600 needs no union (0 orphans, verified).

pm30700 stays full — 213 of its rows have no parent in either table, so
no changed-set join can reach them, and it only costs 13s. pm20100 and
pm00400 stay full because open and index tables delete rows, which a
delete-by-key incremental structurally cannot propagate (cf. icstt).

391s -> 15s. reconcile.py --super-quick reports IN SYNC for all three.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-04 00:21:59 -04:00

570 lines
14 KiB
JSON
Raw Blame History

{
"name": "pm30200",
"source_connection": "sqlserver",
"dest_connection": "usmidsap02",
"dest_table": "gp.pm30200",
"staging_table": "pipekit_staging.pm30200",
"merge_strategy": "incremental",
"merge_key": "vchrnmbr, doctype",
"enabled": 1,
"dest_description": "Dynamics GP payables history",
"columns": [
{
"source_name": "VCHRNMBR",
"source_type": "CHAR(21)",
"dest_name": "vchrnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "VENDORID",
"source_type": "CHAR(15)",
"dest_name": "vendorid",
"dest_type": "text",
"description": null
},
{
"source_name": "DOCTYPE",
"source_type": "SMALLINT",
"dest_name": "doctype",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DOCDATE",
"source_type": "DATETIME",
"dest_name": "docdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "DOCNUMBR",
"source_type": "CHAR(21)",
"dest_name": "docnumbr",
"dest_type": "text",
"description": null
},
{
"source_name": "DOCAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "docamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "CURTRXAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "curtrxam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISTKNAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "distknam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISCAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "discamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DSCDLRAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "dscdlram",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "BACHNUMB",
"source_type": "CHAR(15)",
"dest_name": "bachnumb",
"dest_type": "text",
"description": null
},
{
"source_name": "TRXSORCE",
"source_type": "CHAR(13)",
"dest_name": "trxsorce",
"dest_type": "text",
"description": null
},
{
"source_name": "BCHSOURC",
"source_type": "CHAR(15)",
"dest_name": "bchsourc",
"dest_type": "text",
"description": null
},
{
"source_name": "DISCDATE",
"source_type": "DATETIME",
"dest_name": "discdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "DUEDATE",
"source_type": "DATETIME",
"dest_name": "duedate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "PORDNMBR",
"source_type": "CHAR(21)",
"dest_name": "pordnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "TEN99AMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "ten99amnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "WROFAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "wrofamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISAMTAV",
"source_type": "NUMERIC(19,5)",
"dest_name": "disamtav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TRXDSCRN",
"source_type": "CHAR(31)",
"dest_name": "trxdscrn",
"dest_type": "text",
"description": null
},
{
"source_name": "UN1099AM",
"source_type": "NUMERIC(19,5)",
"dest_name": "un1099am",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "BKTPURAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "bktpuram",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "BKTFRTAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "bktfrtam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "BKTMSCAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "bktmscam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "VOIDED",
"source_type": "TINYINT",
"dest_name": "voided",
"dest_type": "smallint",
"description": null
},
{
"source_name": "HOLD",
"source_type": "TINYINT",
"dest_name": "hold",
"dest_type": "smallint",
"description": null
},
{
"source_name": "CHEKBKID",
"source_type": "CHAR(15)",
"dest_name": "chekbkid",
"dest_type": "text",
"description": null
},
{
"source_name": "DINVPDOF",
"source_type": "DATETIME",
"dest_name": "dinvpdof",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "PPSAMDED",
"source_type": "NUMERIC(19,5)",
"dest_name": "ppsamded",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "PPSTAXRT",
"source_type": "SMALLINT",
"dest_name": "ppstaxrt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "PGRAMSBJ",
"source_type": "SMALLINT",
"dest_name": "pgramsbj",
"dest_type": "smallint",
"description": null
},
{
"source_name": "GSTDSAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "gstdsamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "POSTEDDT",
"source_type": "DATETIME",
"dest_name": "posteddt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "PTDUSRID",
"source_type": "CHAR(15)",
"dest_name": "ptdusrid",
"dest_type": "text",
"description": null
},
{
"source_name": "MODIFDT",
"source_type": "DATETIME",
"dest_name": "modifdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "MDFUSRID",
"source_type": "CHAR(15)",
"dest_name": "mdfusrid",
"dest_type": "text",
"description": null
},
{
"source_name": "PYENTTYP",
"source_type": "SMALLINT",
"dest_name": "pyenttyp",
"dest_type": "smallint",
"description": null
},
{
"source_name": "CARDNAME",
"source_type": "CHAR(15)",
"dest_name": "cardname",
"dest_type": "text",
"description": null
},
{
"source_name": "PRCHAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "prchamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TRDISAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "trdisamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "MSCCHAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "mscchamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "FRTAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "frtamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TAXAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "taxamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TTLPYMTS",
"source_type": "NUMERIC(19,5)",
"dest_name": "ttlpymts",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "CURNCYID",
"source_type": "CHAR(15)",
"dest_name": "curncyid",
"dest_type": "text",
"description": null
},
{
"source_name": "PYMTRMID",
"source_type": "CHAR(21)",
"dest_name": "pymtrmid",
"dest_type": "text",
"description": null
},
{
"source_name": "SHIPMTHD",
"source_type": "CHAR(15)",
"dest_name": "shipmthd",
"dest_type": "text",
"description": null
},
{
"source_name": "TAXSCHID",
"source_type": "CHAR(15)",
"dest_name": "taxschid",
"dest_type": "text",
"description": null
},
{
"source_name": "PCHSCHID",
"source_type": "CHAR(15)",
"dest_name": "pchschid",
"dest_type": "text",
"description": null
},
{
"source_name": "FRTSCHID",
"source_type": "CHAR(15)",
"dest_name": "frtschid",
"dest_type": "text",
"description": null
},
{
"source_name": "MSCSCHID",
"source_type": "CHAR(15)",
"dest_name": "mscschid",
"dest_type": "text",
"description": null
},
{
"source_name": "PSTGDATE",
"source_type": "DATETIME",
"dest_name": "pstgdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "DISAVTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "disavtkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "CNTRLTYP",
"source_type": "SMALLINT",
"dest_name": "cntrltyp",
"dest_type": "smallint",
"description": null
},
{
"source_name": "NOTEINDX",
"source_type": "NUMERIC(19,5)",
"dest_name": "noteindx",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "PRCTDISC",
"source_type": "SMALLINT",
"dest_name": "prctdisc",
"dest_type": "smallint",
"description": null
},
{
"source_name": "RETNAGAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "retnagam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "VOIDPDATE",
"source_type": "DATETIME",
"dest_name": "voidpdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ICTRX",
"source_type": "TINYINT",
"dest_name": "ictrx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "Tax_Date",
"source_type": "DATETIME",
"dest_name": "tax_date",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "PRCHDATE",
"source_type": "DATETIME",
"dest_name": "prchdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "CORRCTN",
"source_type": "TINYINT",
"dest_name": "corrctn",
"dest_type": "smallint",
"description": null
},
{
"source_name": "SIMPLIFD",
"source_type": "TINYINT",
"dest_name": "simplifd",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APLYWITH",
"source_type": "TINYINT",
"dest_name": "aplywith",
"dest_type": "smallint",
"description": null
},
{
"source_name": "Electronic",
"source_type": "TINYINT",
"dest_name": "electronic",
"dest_type": "smallint",
"description": null
},
{
"source_name": "ECTRX",
"source_type": "TINYINT",
"dest_name": "ectrx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DocPrinted",
"source_type": "TINYINT",
"dest_name": "docprinted",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TaxInvReqd",
"source_type": "TINYINT",
"dest_name": "taxinvreqd",
"dest_type": "smallint",
"description": null
},
{
"source_name": "VNDCHKNM",
"source_type": "CHAR(65)",
"dest_name": "vndchknm",
"dest_type": "text",
"description": null
},
{
"source_name": "BackoutTradeDisc",
"source_type": "NUMERIC(19,5)",
"dest_name": "backouttradedisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "CBVAT",
"source_type": "TINYINT",
"dest_name": "cbvat",
"dest_type": "smallint",
"description": null
},
{
"source_name": "VADCDTRO",
"source_type": "CHAR(15)",
"dest_name": "vadcdtro",
"dest_type": "text",
"description": null
},
{
"source_name": "TEN99TYPE",
"source_type": "SMALLINT",
"dest_name": "ten99type",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TEN99BOXNUMBER",
"source_type": "SMALLINT",
"dest_name": "ten99boxnumber",
"dest_type": "smallint",
"description": null
},
{
"source_name": "PONUMBER",
"source_type": "CHAR(17)",
"dest_name": "ponumber",
"dest_type": "text",
"description": null
},
{
"source_name": "Workflow_Status",
"source_type": "SMALLINT",
"dest_name": "workflow_status",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEX_ROW_TS",
"source_type": "DATETIME",
"dest_name": "dex_row_ts",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "DEX_ROW_ID",
"source_type": "INT",
"dest_name": "dex_row_id",
"dest_type": "integer",
"description": null
}
],
"watermarks": [
{
"name": "pm30200_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
"default_value": "1900-01-01"
}
],
"hooks": [],
"description": "Dynamics GP payables history"
}
Reference in New Issue View Git Blame Copy Permalink