pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make the three AP history modules incremental

Browse Source 
The Accounts Payable group ran ~9.5 min, 83% of it in four full
reloads. The cost is the GPSERVER linked-server hop, not row volume
(pm30200 managed only ~590 rows/sec), so cutting rows crossing the link
is the whole win. Predicates go inside OPENQUERY to push down to the
remote — a 4-part name would drag the table across before filtering.

  pm30200  own DEX_ROW_TS
  pm30600  changed set: parent PM30200
  pm30300  changed set: PM30200 union PM20000

All three key on (vchrnmbr, doctype), verified exactly unique in
PM30200 at 108,803/108,803. Watermark resolves off gp.pm30200 with a
7-day lookback, so run order within the group must stay pm30200 first
and the lookback must exceed the sync interval.

The two-parent union on pm30300 is load-bearing: 38 rows have a parent
only in the open table, and a PM30200-only join would strand them
permanently. pm30600 needs no union (0 orphans, verified).

pm30700 stays full — 213 of its rows have no parent in either table, so
no changed-set join can reach them, and it only costs 13s. pm20100 and
pm00400 stay full because open and index tables delete rows, which a
delete-by-key incremental structurally cannot propagate (cf. icstt).

391s -> 15s. reconcile.py --super-quick reports IN SYNC for all three.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-09-04 00:21:59 -04:00
parent e8a3acaebd
commit b855ab91d7
6 changed files with 142 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
config/modules/pm30200.json
View File

@ -4,8 +4,8 @@
"dest_connection": "usmidsap02", "dest_connection": "usmidsap02",
"dest_table": "gp.pm30200", "dest_table": "gp.pm30200",
"staging_table": "pipekit_staging.pm30200", "staging_table": "pipekit_staging.pm30200",
"merge_strategy": "full", "merge_strategy": "incremental",
"merge_key": null, "merge_key": "vchrnmbr, doctype",
"enabled": 1, "enabled": 1,
"dest_description": "Dynamics GP payables history", "dest_description": "Dynamics GP payables history",
"columns": [ "columns": [
@ -556,7 +556,14 @@
"description": null "description": null
} }
], ],
"watermarks": [], "watermarks": [
{
"name": "pm30200_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
"default_value": "1900-01-01"
}
],
"hooks": [], "hooks": [],
"description": "Dynamics GP payables history" "description": "Dynamics GP payables history"
} }

5
config/modules/pm30200.sql
View File

@ -1,3 +1,4 @@
SELECT * FROM OPENQUERY(GPSERVER, '
SELECT SELECT
RTRIM([VCHRNMBR]) AS vchrnmbr, RTRIM([VCHRNMBR]) AS vchrnmbr,
RTRIM([VENDORID]) AS vendorid, RTRIM([VENDORID]) AS vendorid,
@ -77,4 +78,6 @@ SELECT
[Workflow_Status] AS workflow_status, [Workflow_Status] AS workflow_status,
[DEX_ROW_TS] AS dex_row_ts, [DEX_ROW_TS] AS dex_row_ts,
[DEX_ROW_ID] AS dex_row_id [DEX_ROW_ID] AS dex_row_id
FROM [GPSERVER].[CHG].[dbo].[PM30200] FROM CHG.dbo.PM30200
WHERE [DEX_ROW_TS] > ''{pm30200_ts}''
')

13
config/modules/pm30300.json
View File

@ -4,8 +4,8 @@
"dest_connection": "usmidsap02", "dest_connection": "usmidsap02",
"dest_table": "gp.pm30300", "dest_table": "gp.pm30300",
"staging_table": "pipekit_staging.pm30300", "staging_table": "pipekit_staging.pm30300",
"merge_strategy": "full", "merge_strategy": "incremental",
"merge_key": null, "merge_key": "vchrnmbr, doctype",
"enabled": 1, "enabled": 1,
"dest_description": "Dynamics GP payables apply history", "dest_description": "Dynamics GP payables apply history",
"columns": [ "columns": [
@ -458,7 +458,14 @@
"description": null "description": null
} }
], ],
"watermarks": [], "watermarks": [
{
"name": "pm30300_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
"default_value": "1900-01-01"
}
],
"hooks": [], "hooks": [],
"description": "Dynamics GP payables apply history" "description": "Dynamics GP payables apply history"
} }

142
config/modules/pm30300.sql
View File

@ -1,66 +1,78 @@
SELECT * FROM OPENQUERY(GPSERVER, '
SELECT SELECT
RTRIM([VENDORID]) AS vendorid, RTRIM(d.[VENDORID]) AS vendorid,
[DOCDATE] AS docdate, d.[DOCDATE] AS docdate,
[DATE1] AS date1, d.[DATE1] AS date1,
[GLPOSTDT] AS glpostdt, d.[GLPOSTDT] AS glpostdt,
[TIME1] AS time1, d.[TIME1] AS time1,
RTRIM([VCHRNMBR]) AS vchrnmbr, RTRIM(d.[VCHRNMBR]) AS vchrnmbr,
[DOCTYPE] AS doctype, d.[DOCTYPE] AS doctype,
RTRIM([APFRDCNM]) AS apfrdcnm, RTRIM(d.[APFRDCNM]) AS apfrdcnm,
[ApplyFromGLPostDate] AS applyfromglpostdate, d.[ApplyFromGLPostDate] AS applyfromglpostdate,
RTRIM([FROMCURR]) AS fromcurr, RTRIM(d.[FROMCURR]) AS fromcurr,
[APFRMAPLYAMT] AS apfrmaplyamt, d.[APFRMAPLYAMT] AS apfrmaplyamt,
[APFRMDISCTAKEN] AS apfrmdisctaken, d.[APFRMDISCTAKEN] AS apfrmdisctaken,
[APFRMDISCAVAIL] AS apfrmdiscavail, d.[APFRMDISCAVAIL] AS apfrmdiscavail,
[APFRMWROFAMT] AS apfrmwrofamt, d.[APFRMWROFAMT] AS apfrmwrofamt,
[ActualApplyToAmount] AS actualapplytoamount, d.[ActualApplyToAmount] AS actualapplytoamount,
[ActualDiscTakenAmount] AS actualdisctakenamount, d.[ActualDiscTakenAmount] AS actualdisctakenamount,
[ActualDiscAvailTaken] AS actualdiscavailtaken, d.[ActualDiscAvailTaken] AS actualdiscavailtaken,
[ActualWriteOffAmount] AS actualwriteoffamount, d.[ActualWriteOffAmount] AS actualwriteoffamount,
[APFRMEXRATE] AS apfrmexrate, d.[APFRMEXRATE] AS apfrmexrate,
[APFRMDENRATE] AS apfrmdenrate, d.[APFRMDENRATE] AS apfrmdenrate,
[APFRMRTCLCMETH] AS apfrmrtclcmeth, d.[APFRMRTCLCMETH] AS apfrmrtclcmeth,
[APFRMMCTRXSTT] AS apfrmmctrxstt, d.[APFRMMCTRXSTT] AS apfrmmctrxstt,
RTRIM([APTVCHNM]) AS aptvchnm, RTRIM(d.[APTVCHNM]) AS aptvchnm,
[APTODCTY] AS aptodcty, d.[APTODCTY] AS aptodcty,
RTRIM([APTODCNM]) AS aptodcnm, RTRIM(d.[APTODCNM]) AS aptodcnm,
[APTODCDT] AS aptodcdt, d.[APTODCDT] AS aptodcdt,
[ApplyToGLPostDate] AS applytoglpostdate, d.[ApplyToGLPostDate] AS applytoglpostdate,
RTRIM([CURNCYID]) AS curncyid, RTRIM(d.[CURNCYID]) AS curncyid,
[CURRNIDX] AS currnidx, d.[CURRNIDX] AS currnidx,
[APPLDAMT] AS appldamt, d.[APPLDAMT] AS appldamt,
[DISTKNAM] AS distknam, d.[DISTKNAM] AS distknam,
[DISAVTKN] AS disavtkn, d.[DISAVTKN] AS disavtkn,
[WROFAMNT] AS wrofamnt, d.[WROFAMNT] AS wrofamnt,
[ORAPPAMT] AS orappamt, d.[ORAPPAMT] AS orappamt,
[ORDISTKN] AS ordistkn, d.[ORDISTKN] AS ordistkn,
[ORDATKN] AS ordatkn, d.[ORDATKN] AS ordatkn,
[ORWROFAM] AS orwrofam, d.[ORWROFAM] AS orwrofam,
[APTOEXRATE] AS aptoexrate, d.[APTOEXRATE] AS aptoexrate,
[APTODENRATE] AS aptodenrate, d.[APTODENRATE] AS aptodenrate,
[APTORTCLCMETH] AS aptortclcmeth, d.[APTORTCLCMETH] AS aptortclcmeth,
[APTOMCTRXSTT] AS aptomctrxstt, d.[APTOMCTRXSTT] AS aptomctrxstt,
[PPSAMDED] AS ppsamded, d.[PPSAMDED] AS ppsamded,
[GSTDSAMT] AS gstdsamt, d.[GSTDSAMT] AS gstdsamt,
RTRIM([TAXDTLID]) AS taxdtlid, RTRIM(d.[TAXDTLID]) AS taxdtlid,
[POSTED] AS posted, d.[POSTED] AS posted,
[TEN99AMNT] AS ten99amnt, d.[TEN99AMNT] AS ten99amnt,
[RLGANLOS] AS rlganlos, d.[RLGANLOS] AS rlganlos,
[APYFRMRNDAMT] AS apyfrmrndamt, d.[APYFRMRNDAMT] AS apyfrmrndamt,
[APYTORNDAMT] AS apytorndamt, d.[APYTORNDAMT] AS apytorndamt,
[APYTORNDDISC] AS apytornddisc, d.[APYTORNDDISC] AS apytornddisc,
[OAPYFRMRNDAMT] AS oapyfrmrndamt, d.[OAPYFRMRNDAMT] AS oapyfrmrndamt,
[OAPYTORNDAMT] AS oapytorndamt, d.[OAPYTORNDAMT] AS oapytorndamt,
[OAPYTORNDDISC] AS oapytornddisc, d.[OAPYTORNDDISC] AS oapytornddisc,
[Settled_Gain_CreditCurrT] AS settled_gain_creditcurrt, d.[Settled_Gain_CreditCurrT] AS settled_gain_creditcurrt,
[Settled_Loss_CreditCurrT] AS settled_loss_creditcurrt, d.[Settled_Loss_CreditCurrT] AS settled_loss_creditcurrt,
[Settled_Gain_DebitCurrTr] AS settled_gain_debitcurrtr, d.[Settled_Gain_DebitCurrTr] AS settled_gain_debitcurrtr,
[Settled_Loss_DebitCurrTr] AS settled_loss_debitcurrtr, d.[Settled_Loss_DebitCurrTr] AS settled_loss_debitcurrtr,
[Settled_Gain_DebitDiscAv] AS settled_gain_debitdiscav, d.[Settled_Gain_DebitDiscAv] AS settled_gain_debitdiscav,
[Settled_Loss_DebitDiscAv] AS settled_loss_debitdiscav, d.[Settled_Loss_DebitDiscAv] AS settled_loss_debitdiscav,
[Revaluation_Status] AS revaluation_status, d.[Revaluation_Status] AS revaluation_status,
[Credit1099Amount] AS credit1099amount, d.[Credit1099Amount] AS credit1099amount,
[DEFTEN99TYPE] AS deften99type, d.[DEFTEN99TYPE] AS deften99type,
[DEFTEN99BOXNUMBER] AS deften99boxnumber, d.[DEFTEN99BOXNUMBER] AS deften99boxnumber,
[DEX_ROW_ID] AS dex_row_id d.[DEX_ROW_ID] AS dex_row_id
FROM [GPSERVER].[CHG].[dbo].[PM30300] FROM CHG.dbo.PM30300 d
WHERE EXISTS (
SELECT 1 FROM CHG.dbo.PM30200 p
WHERE p.VCHRNMBR = d.VCHRNMBR AND p.DOCTYPE = d.DOCTYPE
AND p.[DEX_ROW_TS] > ''{pm30300_ts}''
)
OR EXISTS (
SELECT 1 FROM CHG.dbo.PM20000 o
WHERE o.VCHRNMBR = d.VCHRNMBR AND o.DOCTYPE = d.DOCTYPE
AND o.[DEX_ROW_TS] > ''{pm30300_ts}''
)
')

13
config/modules/pm30600.json
View File

@ -4,8 +4,8 @@
"dest_connection": "usmidsap02", "dest_connection": "usmidsap02",
"dest_table": "gp.pm30600", "dest_table": "gp.pm30600",
"staging_table": "pipekit_staging.pm30600", "staging_table": "pipekit_staging.pm30600",
"merge_strategy": "full", "merge_strategy": "incremental",
"merge_key": null, "merge_key": "vchrnmbr, doctype",
"enabled": 1, "enabled": 1,
"dest_description": "Dynamics GP payables GL distribution history", "dest_description": "Dynamics GP payables GL distribution history",
"columns": [ "columns": [
@ -171,7 +171,14 @@
"description": null "description": null
} }
], ],
"watermarks": [], "watermarks": [
{
"name": "pm30600_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '7 days', '1900-01-01') FROM gp.pm30200",
"default_value": "1900-01-01"
}
],
"hooks": [], "hooks": [],
"description": "Dynamics GP payables GL distribution history" "description": "Dynamics GP payables GL distribution history"
} }

55
config/modules/pm30600.sql
View File

@ -1,25 +1,32 @@
SELECT * FROM OPENQUERY(GPSERVER, '
SELECT SELECT
[DOCTYPE] AS doctype, d.[DOCTYPE] AS doctype,
RTRIM([VCHRNMBR]) AS vchrnmbr, RTRIM(d.[VCHRNMBR]) AS vchrnmbr,
[DSTSQNUM] AS dstsqnum, d.[DSTSQNUM] AS dstsqnum,
[CNTRLTYP] AS cntrltyp, d.[CNTRLTYP] AS cntrltyp,
[CRDTAMNT] AS crdtamnt, d.[CRDTAMNT] AS crdtamnt,
[DEBITAMT] AS debitamt, d.[DEBITAMT] AS debitamt,
[DSTINDX] AS dstindx, d.[DSTINDX] AS dstindx,
[DISTTYPE] AS disttype, d.[DISTTYPE] AS disttype,
[CHANGED] AS changed, d.[CHANGED] AS changed,
RTRIM([USERID]) AS userid, RTRIM(d.[USERID]) AS userid,
[PSTGSTUS] AS pstgstus, d.[PSTGSTUS] AS pstgstus,
RTRIM([VENDORID]) AS vendorid, RTRIM(d.[VENDORID]) AS vendorid,
RTRIM([TRXSORCE]) AS trxsorce, RTRIM(d.[TRXSORCE]) AS trxsorce,
[PSTGDATE] AS pstgdate, d.[PSTGDATE] AS pstgdate,
RTRIM([CURNCYID]) AS curncyid, RTRIM(d.[CURNCYID]) AS curncyid,
[CURRNIDX] AS currnidx, d.[CURRNIDX] AS currnidx,
[ORCRDAMT] AS orcrdamt, d.[ORCRDAMT] AS orcrdamt,
[ORDBTAMT] AS ordbtamt, d.[ORDBTAMT] AS ordbtamt,
RTRIM([APTVCHNM]) AS aptvchnm, RTRIM(d.[APTVCHNM]) AS aptvchnm,
[APTODCTY] AS aptodcty, d.[APTODCTY] AS aptodcty,
[SPCLDIST] AS spcldist, d.[SPCLDIST] AS spcldist,
RTRIM([DistRef]) AS distref, RTRIM(d.[DistRef]) AS distref,
[DEX_ROW_ID] AS dex_row_id d.[DEX_ROW_ID] AS dex_row_id
FROM [GPSERVER].[CHG].[dbo].[PM30600] FROM CHG.dbo.PM30600 d
WHERE EXISTS (
SELECT 1 FROM CHG.dbo.PM30200 p
WHERE p.VCHRNMBR = d.VCHRNMBR AND p.DOCTYPE = d.DOCTYPE
AND p.[DEX_ROW_TS] > ''{pm30600_ts}''
)
')