pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
1d37f27573
pipekit/config/modules/pm30300.json
Paul Trowbridge b855ab91d7 Make the three AP history modules incremental 
The Accounts Payable group ran ~9.5 min, 83% of it in four full
reloads. The cost is the GPSERVER linked-server hop, not row volume
(pm30200 managed only ~590 rows/sec), so cutting rows crossing the link
is the whole win. Predicates go inside OPENQUERY to push down to the
remote — a 4-part name would drag the table across before filtering.

  pm30200  own DEX_ROW_TS
  pm30600  changed set: parent PM30200
  pm30300  changed set: PM30200 union PM20000

All three key on (vchrnmbr, doctype), verified exactly unique in
PM30200 at 108,803/108,803. Watermark resolves off gp.pm30200 with a
7-day lookback, so run order within the group must stay pm30200 first
and the lookback must exceed the sync interval.

The two-parent union on pm30300 is load-bearing: 38 rows have a parent
only in the open table, and a PM30200-only join would strand them
permanently. pm30600 needs no union (0 orphans, verified).

pm30700 stays full — 213 of its rows have no parent in either table, so
no changed-set join can reach them, and it only costs 13s. pm20100 and
pm00400 stay full because open and index tables delete rows, which a
delete-by-key incremental structurally cannot propagate (cf. icstt).

391s -> 15s. reconcile.py --super-quick reports IN SYNC for all three.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-04 00:21:59 -04:00

472 lines
12 KiB
JSON
Raw Blame History

{
"name": "pm30300",
"source_connection": "sqlserver",
"dest_connection": "usmidsap02",
"dest_table": "gp.pm30300",
"staging_table": "pipekit_staging.pm30300",
"merge_strategy": "incremental",
"merge_key": "vchrnmbr, doctype",
"enabled": 1,
"dest_description": "Dynamics GP payables apply history",
"columns": [
{
"source_name": "VENDORID",
"source_type": "CHAR(15)",
"dest_name": "vendorid",
"dest_type": "text",
"description": null
},
{
"source_name": "DOCDATE",
"source_type": "DATETIME",
"dest_name": "docdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "DATE1",
"source_type": "DATETIME",
"dest_name": "date1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "GLPOSTDT",
"source_type": "DATETIME",
"dest_name": "glpostdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "TIME1",
"source_type": "DATETIME",
"dest_name": "time1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "VCHRNMBR",
"source_type": "CHAR(21)",
"dest_name": "vchrnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "DOCTYPE",
"source_type": "SMALLINT",
"dest_name": "doctype",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRDCNM",
"source_type": "CHAR(21)",
"dest_name": "apfrdcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "ApplyFromGLPostDate",
"source_type": "DATETIME",
"dest_name": "applyfromglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "FROMCURR",
"source_type": "CHAR(15)",
"dest_name": "fromcurr",
"dest_type": "text",
"description": null
},
{
"source_name": "APFRMAPLYAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmaplyamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCTAKEN",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdisctaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCAVAIL",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdiscavail",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMWROFAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmwrofamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualApplyToAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualapplytoamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscTakenAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdisctakenamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscAvailTaken",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdiscavailtaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualWriteOffAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualwriteoffamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMDENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmdenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMRTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "apfrmrtclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRMMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "apfrmmctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTVCHNM",
"source_type": "CHAR(21)",
"dest_name": "aptvchnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCTY",
"source_type": "SMALLINT",
"dest_name": "aptodcty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTODCNM",
"source_type": "CHAR(21)",
"dest_name": "aptodcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCDT",
"source_type": "DATETIME",
"dest_name": "aptodcdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ApplyToGLPostDate",
"source_type": "DATETIME",
"dest_name": "applytoglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "CURNCYID",
"source_type": "CHAR(15)",
"dest_name": "curncyid",
"dest_type": "text",
"description": null
},
{
"source_name": "CURRNIDX",
"source_type": "SMALLINT",
"dest_name": "currnidx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APPLDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "appldamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISTKNAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "distknam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISAVTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "disavtkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "WROFAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "wrofamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORAPPAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "orappamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDISTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordistkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDATKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordatkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORWROFAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "orwrofam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APTOEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptoexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTODENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptodenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTORTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "aptortclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTOMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "aptomctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "PPSAMDED",
"source_type": "NUMERIC(19,5)",
"dest_name": "ppsamded",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "GSTDSAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "gstdsamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TAXDTLID",
"source_type": "CHAR(15)",
"dest_name": "taxdtlid",
"dest_type": "text",
"description": null
},
{
"source_name": "POSTED",
"source_type": "TINYINT",
"dest_name": "posted",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TEN99AMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "ten99amnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "RLGANLOS",
"source_type": "NUMERIC(19,5)",
"dest_name": "rlganlos",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Revaluation_Status",
"source_type": "SMALLINT",
"dest_name": "revaluation_status",
"dest_type": "smallint",
"description": null
},
{
"source_name": "Credit1099Amount",
"source_type": "NUMERIC(19,5)",
"dest_name": "credit1099amount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DEFTEN99TYPE",
"source_type": "SMALLINT",
"dest_name": "deften99type",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEFTEN99BOXNUMBER",
"source_type": "SMALLINT",
"dest_name": "deften99boxnumber",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEX_ROW_ID",
"source_type": "INT",
"dest_name": "dex_row_id",
"dest_type": "integer",
"description": null
}
],
"watermarks": [
{
"name": "pm30300_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
"default_value": "1900-01-01"
}
],
"hooks": [],
"description": "Dynamics GP payables apply history"
}
Reference in New Issue View Git Blame Copy Permalink