Make pm20100 incremental on MODIFDT
My earlier reasoning for leaving this full — "open tables delete rows, so delete-by-key strands them" — does not hold for PM20100. The table accumulates rather than sheds: 46,841 of its 46,886 rows belong to a document already in PM30200 (history), and 46,864 also appear in PM30300. MODIFDT spans 2017-2026 and is populated on every row. The changed set is self-referential rather than parent-driven: only 42 of 46,886 rows have a parent in PM20000, so a PM20000 join would sync almost nothing. Instead, stage every apply row belonging to a voucher that has any row modified since the watermark, and key the merge on (vchrnmbr, doctype) so the delete replaces the voucher's whole apply set. The natural grain (vchrnmbr, doctype, aptvchnm, aptodcty) has 42 duplicates and would drop rows if used as the key directly. 7-day lookback stages 60 rows (738 at 30 days) against 46,886 for a full reload. 84s -> 5s. reconcile.py --super-quick reports IN SYNC. The watermark guards MAX(modifdt) with LEAST(..., CURRENT_DATE) so a future-dated row cannot advance it past real data. Residual risk, same class as icstt: an upstream delete cannot propagate. Low here given the table demonstrably retains rows whose documents went historical years ago. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b855ab91d7
commit
855dd1a402
@ -4,8 +4,8 @@
|
||||
"dest_connection": "usmidsap02",
|
||||
"dest_table": "gp.pm20100",
|
||||
"staging_table": "pipekit_staging.pm20100",
|
||||
"merge_strategy": "full",
|
||||
"merge_key": null,
|
||||
"merge_strategy": "incremental",
|
||||
"merge_key": "vchrnmbr, doctype",
|
||||
"enabled": 1,
|
||||
"dest_description": "Dynamics GP open payables apply detail",
|
||||
"columns": [
|
||||
@ -626,7 +626,14 @@
|
||||
"description": null
|
||||
}
|
||||
],
|
||||
"watermarks": [],
|
||||
"watermarks": [
|
||||
{
|
||||
"name": "pm20100_ts",
|
||||
"connection": "usmidsap02",
|
||||
"resolver_sql": "SELECT COALESCE(LEAST(MAX(modifdt), CURRENT_DATE::timestamp) - INTERVAL '7 days', '1900-01-01') FROM gp.pm20100",
|
||||
"default_value": "1900-01-01"
|
||||
}
|
||||
],
|
||||
"hooks": [],
|
||||
"description": "Dynamics GP open payables apply detail"
|
||||
}
|
||||
|
||||
@ -1,90 +1,97 @@
|
||||
SELECT * FROM OPENQUERY(GPSERVER, '
|
||||
SELECT
|
||||
RTRIM([KEYSOURC]) AS keysourc,
|
||||
[DOCDATE] AS docdate,
|
||||
[TIME1] AS time1,
|
||||
RTRIM([VENDORID]) AS vendorid,
|
||||
[POSTED] AS posted,
|
||||
RTRIM([BCHSOURC]) AS bchsourc,
|
||||
RTRIM([USERID]) AS userid,
|
||||
RTRIM([APTVCHNM]) AS aptvchnm,
|
||||
[APTODCTY] AS aptodcty,
|
||||
[APTODCDT] AS aptodcdt,
|
||||
RTRIM([APTODCNM]) AS aptodcnm,
|
||||
[APPLDAMT] AS appldamt,
|
||||
[DISTKNAM] AS distknam,
|
||||
[DISAVTKN] AS disavtkn,
|
||||
[WROFAMNT] AS wrofamnt,
|
||||
RTRIM([CURNCYID]) AS curncyid,
|
||||
[CURRNIDX] AS currnidx,
|
||||
[ORAPPAMT] AS orappamt,
|
||||
[ORDISTKN] AS ordistkn,
|
||||
[ORDATKN] AS ordatkn,
|
||||
[ORWROFAM] AS orwrofam,
|
||||
[Apply_To_Document_Amount] AS apply_to_document_amount,
|
||||
[Apply_To_Orig_Doc_Amount] AS apply_to_orig_doc_amount,
|
||||
[Apply_To_Discount_Date] AS apply_to_discount_date,
|
||||
[Apply_To_Due_Date] AS apply_to_due_date,
|
||||
RTRIM([Apply_To_Description]) AS apply_to_description,
|
||||
[Apply_To_IS_MC_Trx] AS apply_to_is_mc_trx,
|
||||
[ApplyToGLPostDate] AS applytoglpostdate,
|
||||
[APTOEXRATE] AS aptoexrate,
|
||||
[APTODENRATE] AS aptodenrate,
|
||||
[APTORTCLCMETH] AS aptortclcmeth,
|
||||
[APTOMCTRXSTT] AS aptomctrxstt,
|
||||
[ISMCTRX] AS ismctrx,
|
||||
RTRIM([VCHRNMBR]) AS vchrnmbr,
|
||||
[DOCTYPE] AS doctype,
|
||||
RTRIM([APFRDCNM]) AS apfrdcnm,
|
||||
RTRIM([FROMCURR]) AS fromcurr,
|
||||
[Apply_From_Document_Amou] AS apply_from_document_amou,
|
||||
[Apply_From_Orig_Doc_Amou] AS apply_from_orig_doc_amou,
|
||||
[APFRMAPLYAMT] AS apfrmaplyamt,
|
||||
[APFRMDISCTAKEN] AS apfrmdisctaken,
|
||||
[APFRMDISCAVAIL] AS apfrmdiscavail,
|
||||
[APFRMWROFAMT] AS apfrmwrofamt,
|
||||
[ActualApplyToAmount] AS actualapplytoamount,
|
||||
[ActualDiscTakenAmount] AS actualdisctakenamount,
|
||||
[ActualDiscAvailTaken] AS actualdiscavailtaken,
|
||||
[ActualWriteOffAmount] AS actualwriteoffamount,
|
||||
[APFRMEXRATE] AS apfrmexrate,
|
||||
[APFRMDENRATE] AS apfrmdenrate,
|
||||
[APFRMRTCLCMETH] AS apfrmrtclcmeth,
|
||||
[APFRMMCTRXSTT] AS apfrmmctrxstt,
|
||||
RTRIM([Apply_From_Description]) AS apply_from_description,
|
||||
[Apply_From_IS_MC_Trx] AS apply_from_is_mc_trx,
|
||||
[PPSAMDED] AS ppsamded,
|
||||
[GSTDSAMT] AS gstdsamt,
|
||||
RTRIM([TAXDTLID]) AS taxdtlid,
|
||||
[UPDTKNAM] AS updtknam,
|
||||
[UPGSTDAM] AS upgstdam,
|
||||
[UNPPPSAD] AS unpppsad,
|
||||
[UPWROFAM] AS upwrofam,
|
||||
[PMDSTMSG] AS pmdstmsg,
|
||||
[MODIFDT] AS modifdt,
|
||||
RTRIM([MDFUSRID]) AS mdfusrid,
|
||||
[POSTEDDT] AS posteddt,
|
||||
RTRIM([PTDUSRID]) AS ptdusrid,
|
||||
[TEN99AMNT] AS ten99amnt,
|
||||
[RLGANLOS] AS rlganlos,
|
||||
[APYFRMRNDAMT] AS apyfrmrndamt,
|
||||
[APYTORNDAMT] AS apytorndamt,
|
||||
[APYTORNDDISC] AS apytornddisc,
|
||||
[XCHGRATE] AS xchgrate,
|
||||
[DECPLCUR] AS decplcur,
|
||||
[RATECALC] AS ratecalc,
|
||||
RTRIM([Payment_To_Print_On_Stub]) AS payment_to_print_on_stub,
|
||||
[OAPYFRMRNDAMT] AS oapyfrmrndamt,
|
||||
[OAPYTORNDAMT] AS oapytorndamt,
|
||||
[OAPYTORNDDISC] AS oapytornddisc,
|
||||
[Settled_Gain_CreditCurrT] AS settled_gain_creditcurrt,
|
||||
[Settled_Loss_CreditCurrT] AS settled_loss_creditcurrt,
|
||||
[Settled_Gain_DebitCurrTr] AS settled_gain_debitcurrtr,
|
||||
[Settled_Loss_DebitCurrTr] AS settled_loss_debitcurrtr,
|
||||
[Settled_Gain_DebitDiscAv] AS settled_gain_debitdiscav,
|
||||
[Settled_Loss_DebitDiscAv] AS settled_loss_debitdiscav,
|
||||
RTRIM([VADCDTRO]) AS vadcdtro,
|
||||
[Credit1099Amount] AS credit1099amount,
|
||||
[DEFTEN99TYPE] AS deften99type,
|
||||
[DEFTEN99BOXNUMBER] AS deften99boxnumber,
|
||||
[DEX_ROW_ID] AS dex_row_id
|
||||
FROM [GPSERVER].[CHG].[dbo].[PM20100]
|
||||
RTRIM(d.[KEYSOURC]) AS keysourc,
|
||||
d.[DOCDATE] AS docdate,
|
||||
d.[TIME1] AS time1,
|
||||
RTRIM(d.[VENDORID]) AS vendorid,
|
||||
d.[POSTED] AS posted,
|
||||
RTRIM(d.[BCHSOURC]) AS bchsourc,
|
||||
RTRIM(d.[USERID]) AS userid,
|
||||
RTRIM(d.[APTVCHNM]) AS aptvchnm,
|
||||
d.[APTODCTY] AS aptodcty,
|
||||
d.[APTODCDT] AS aptodcdt,
|
||||
RTRIM(d.[APTODCNM]) AS aptodcnm,
|
||||
d.[APPLDAMT] AS appldamt,
|
||||
d.[DISTKNAM] AS distknam,
|
||||
d.[DISAVTKN] AS disavtkn,
|
||||
d.[WROFAMNT] AS wrofamnt,
|
||||
RTRIM(d.[CURNCYID]) AS curncyid,
|
||||
d.[CURRNIDX] AS currnidx,
|
||||
d.[ORAPPAMT] AS orappamt,
|
||||
d.[ORDISTKN] AS ordistkn,
|
||||
d.[ORDATKN] AS ordatkn,
|
||||
d.[ORWROFAM] AS orwrofam,
|
||||
d.[Apply_To_Document_Amount] AS apply_to_document_amount,
|
||||
d.[Apply_To_Orig_Doc_Amount] AS apply_to_orig_doc_amount,
|
||||
d.[Apply_To_Discount_Date] AS apply_to_discount_date,
|
||||
d.[Apply_To_Due_Date] AS apply_to_due_date,
|
||||
RTRIM(d.[Apply_To_Description]) AS apply_to_description,
|
||||
d.[Apply_To_IS_MC_Trx] AS apply_to_is_mc_trx,
|
||||
d.[ApplyToGLPostDate] AS applytoglpostdate,
|
||||
d.[APTOEXRATE] AS aptoexrate,
|
||||
d.[APTODENRATE] AS aptodenrate,
|
||||
d.[APTORTCLCMETH] AS aptortclcmeth,
|
||||
d.[APTOMCTRXSTT] AS aptomctrxstt,
|
||||
d.[ISMCTRX] AS ismctrx,
|
||||
RTRIM(d.[VCHRNMBR]) AS vchrnmbr,
|
||||
d.[DOCTYPE] AS doctype,
|
||||
RTRIM(d.[APFRDCNM]) AS apfrdcnm,
|
||||
RTRIM(d.[FROMCURR]) AS fromcurr,
|
||||
d.[Apply_From_Document_Amou] AS apply_from_document_amou,
|
||||
d.[Apply_From_Orig_Doc_Amou] AS apply_from_orig_doc_amou,
|
||||
d.[APFRMAPLYAMT] AS apfrmaplyamt,
|
||||
d.[APFRMDISCTAKEN] AS apfrmdisctaken,
|
||||
d.[APFRMDISCAVAIL] AS apfrmdiscavail,
|
||||
d.[APFRMWROFAMT] AS apfrmwrofamt,
|
||||
d.[ActualApplyToAmount] AS actualapplytoamount,
|
||||
d.[ActualDiscTakenAmount] AS actualdisctakenamount,
|
||||
d.[ActualDiscAvailTaken] AS actualdiscavailtaken,
|
||||
d.[ActualWriteOffAmount] AS actualwriteoffamount,
|
||||
d.[APFRMEXRATE] AS apfrmexrate,
|
||||
d.[APFRMDENRATE] AS apfrmdenrate,
|
||||
d.[APFRMRTCLCMETH] AS apfrmrtclcmeth,
|
||||
d.[APFRMMCTRXSTT] AS apfrmmctrxstt,
|
||||
RTRIM(d.[Apply_From_Description]) AS apply_from_description,
|
||||
d.[Apply_From_IS_MC_Trx] AS apply_from_is_mc_trx,
|
||||
d.[PPSAMDED] AS ppsamded,
|
||||
d.[GSTDSAMT] AS gstdsamt,
|
||||
RTRIM(d.[TAXDTLID]) AS taxdtlid,
|
||||
d.[UPDTKNAM] AS updtknam,
|
||||
d.[UPGSTDAM] AS upgstdam,
|
||||
d.[UNPPPSAD] AS unpppsad,
|
||||
d.[UPWROFAM] AS upwrofam,
|
||||
d.[PMDSTMSG] AS pmdstmsg,
|
||||
d.[MODIFDT] AS modifdt,
|
||||
RTRIM(d.[MDFUSRID]) AS mdfusrid,
|
||||
d.[POSTEDDT] AS posteddt,
|
||||
RTRIM(d.[PTDUSRID]) AS ptdusrid,
|
||||
d.[TEN99AMNT] AS ten99amnt,
|
||||
d.[RLGANLOS] AS rlganlos,
|
||||
d.[APYFRMRNDAMT] AS apyfrmrndamt,
|
||||
d.[APYTORNDAMT] AS apytorndamt,
|
||||
d.[APYTORNDDISC] AS apytornddisc,
|
||||
d.[XCHGRATE] AS xchgrate,
|
||||
d.[DECPLCUR] AS decplcur,
|
||||
d.[RATECALC] AS ratecalc,
|
||||
RTRIM(d.[Payment_To_Print_On_Stub]) AS payment_to_print_on_stub,
|
||||
d.[OAPYFRMRNDAMT] AS oapyfrmrndamt,
|
||||
d.[OAPYTORNDAMT] AS oapytorndamt,
|
||||
d.[OAPYTORNDDISC] AS oapytornddisc,
|
||||
d.[Settled_Gain_CreditCurrT] AS settled_gain_creditcurrt,
|
||||
d.[Settled_Loss_CreditCurrT] AS settled_loss_creditcurrt,
|
||||
d.[Settled_Gain_DebitCurrTr] AS settled_gain_debitcurrtr,
|
||||
d.[Settled_Loss_DebitCurrTr] AS settled_loss_debitcurrtr,
|
||||
d.[Settled_Gain_DebitDiscAv] AS settled_gain_debitdiscav,
|
||||
d.[Settled_Loss_DebitDiscAv] AS settled_loss_debitdiscav,
|
||||
RTRIM(d.[VADCDTRO]) AS vadcdtro,
|
||||
d.[Credit1099Amount] AS credit1099amount,
|
||||
d.[DEFTEN99TYPE] AS deften99type,
|
||||
d.[DEFTEN99BOXNUMBER] AS deften99boxnumber,
|
||||
d.[DEX_ROW_ID] AS dex_row_id
|
||||
FROM CHG.dbo.PM20100 d
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM CHG.dbo.PM20100 x
|
||||
WHERE x.VCHRNMBR = d.VCHRNMBR AND x.DOCTYPE = d.DOCTYPE
|
||||
AND x.[MODIFDT] > ''{pm20100_ts}''
|
||||
)
|
||||
')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user