pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
1d37f27573
pipekit/config/modules/pm20100.json
Paul Trowbridge 855dd1a402 Make pm20100 incremental on MODIFDT 
My earlier reasoning for leaving this full — "open tables delete rows,
so delete-by-key strands them" — does not hold for PM20100. The table
accumulates rather than sheds: 46,841 of its 46,886 rows belong to a
document already in PM30200 (history), and 46,864 also appear in
PM30300. MODIFDT spans 2017-2026 and is populated on every row.

The changed set is self-referential rather than parent-driven: only 42
of 46,886 rows have a parent in PM20000, so a PM20000 join would sync
almost nothing. Instead, stage every apply row belonging to a voucher
that has any row modified since the watermark, and key the merge on
(vchrnmbr, doctype) so the delete replaces the voucher's whole apply
set. The natural grain (vchrnmbr, doctype, aptvchnm, aptodcty) has 42
duplicates and would drop rows if used as the key directly.

7-day lookback stages 60 rows (738 at 30 days) against 46,886 for a
full reload. 84s -> 5s. reconcile.py --super-quick reports IN SYNC.

The watermark guards MAX(modifdt) with LEAST(..., CURRENT_DATE) so a
future-dated row cannot advance it past real data.

Residual risk, same class as icstt: an upstream delete cannot
propagate. Low here given the table demonstrably retains rows whose
documents went historical years ago.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-04 00:34:43 -04:00

640 lines
16 KiB
JSON
Raw Blame History

{
"name": "pm20100",
"source_connection": "sqlserver",
"dest_connection": "usmidsap02",
"dest_table": "gp.pm20100",
"staging_table": "pipekit_staging.pm20100",
"merge_strategy": "incremental",
"merge_key": "vchrnmbr, doctype",
"enabled": 1,
"dest_description": "Dynamics GP open payables apply detail",
"columns": [
{
"source_name": "KEYSOURC",
"source_type": "CHAR(41)",
"dest_name": "keysourc",
"dest_type": "text",
"description": null
},
{
"source_name": "DOCDATE",
"source_type": "DATETIME",
"dest_name": "docdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "TIME1",
"source_type": "DATETIME",
"dest_name": "time1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "VENDORID",
"source_type": "CHAR(15)",
"dest_name": "vendorid",
"dest_type": "text",
"description": null
},
{
"source_name": "POSTED",
"source_type": "TINYINT",
"dest_name": "posted",
"dest_type": "smallint",
"description": null
},
{
"source_name": "BCHSOURC",
"source_type": "CHAR(15)",
"dest_name": "bchsourc",
"dest_type": "text",
"description": null
},
{
"source_name": "USERID",
"source_type": "CHAR(15)",
"dest_name": "userid",
"dest_type": "text",
"description": null
},
{
"source_name": "APTVCHNM",
"source_type": "CHAR(21)",
"dest_name": "aptvchnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCTY",
"source_type": "SMALLINT",
"dest_name": "aptodcty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTODCDT",
"source_type": "DATETIME",
"dest_name": "aptodcdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "APTODCNM",
"source_type": "CHAR(21)",
"dest_name": "aptodcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APPLDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "appldamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISTKNAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "distknam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISAVTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "disavtkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "WROFAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "wrofamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "CURNCYID",
"source_type": "CHAR(15)",
"dest_name": "curncyid",
"dest_type": "text",
"description": null
},
{
"source_name": "CURRNIDX",
"source_type": "SMALLINT",
"dest_name": "currnidx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "ORAPPAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "orappamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDISTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordistkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDATKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordatkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORWROFAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "orwrofam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Apply_To_Document_Amount",
"source_type": "NUMERIC(19,5)",
"dest_name": "apply_to_document_amount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Apply_To_Orig_Doc_Amount",
"source_type": "NUMERIC(19,5)",
"dest_name": "apply_to_orig_doc_amount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Apply_To_Discount_Date",
"source_type": "DATETIME",
"dest_name": "apply_to_discount_date",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "Apply_To_Due_Date",
"source_type": "DATETIME",
"dest_name": "apply_to_due_date",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "Apply_To_Description",
"source_type": "CHAR(31)",
"dest_name": "apply_to_description",
"dest_type": "text",
"description": null
},
{
"source_name": "Apply_To_IS_MC_Trx",
"source_type": "SMALLINT",
"dest_name": "apply_to_is_mc_trx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "ApplyToGLPostDate",
"source_type": "DATETIME",
"dest_name": "applytoglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "APTOEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptoexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTODENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptodenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTORTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "aptortclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTOMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "aptomctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "ISMCTRX",
"source_type": "SMALLINT",
"dest_name": "ismctrx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "VCHRNMBR",
"source_type": "CHAR(21)",
"dest_name": "vchrnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "DOCTYPE",
"source_type": "SMALLINT",
"dest_name": "doctype",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRDCNM",
"source_type": "CHAR(21)",
"dest_name": "apfrdcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "FROMCURR",
"source_type": "CHAR(15)",
"dest_name": "fromcurr",
"dest_type": "text",
"description": null
},
{
"source_name": "Apply_From_Document_Amou",
"source_type": "NUMERIC(19,5)",
"dest_name": "apply_from_document_amou",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Apply_From_Orig_Doc_Amou",
"source_type": "NUMERIC(19,5)",
"dest_name": "apply_from_orig_doc_amou",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMAPLYAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmaplyamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCTAKEN",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdisctaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCAVAIL",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdiscavail",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMWROFAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmwrofamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualApplyToAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualapplytoamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscTakenAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdisctakenamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscAvailTaken",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdiscavailtaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualWriteOffAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualwriteoffamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMDENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmdenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMRTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "apfrmrtclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRMMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "apfrmmctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "Apply_From_Description",
"source_type": "CHAR(31)",
"dest_name": "apply_from_description",
"dest_type": "text",
"description": null
},
{
"source_name": "Apply_From_IS_MC_Trx",
"source_type": "SMALLINT",
"dest_name": "apply_from_is_mc_trx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "PPSAMDED",
"source_type": "NUMERIC(19,5)",
"dest_name": "ppsamded",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "GSTDSAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "gstdsamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TAXDTLID",
"source_type": "CHAR(15)",
"dest_name": "taxdtlid",
"dest_type": "text",
"description": null
},
{
"source_name": "UPDTKNAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "updtknam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "UPGSTDAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "upgstdam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "UNPPPSAD",
"source_type": "NUMERIC(19,5)",
"dest_name": "unpppsad",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "UPWROFAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "upwrofam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "PMDSTMSG",
"source_type": "BINARY",
"dest_name": "pmdstmsg",
"dest_type": "bytea",
"description": null
},
{
"source_name": "MODIFDT",
"source_type": "DATETIME",
"dest_name": "modifdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "MDFUSRID",
"source_type": "CHAR(15)",
"dest_name": "mdfusrid",
"dest_type": "text",
"description": null
},
{
"source_name": "POSTEDDT",
"source_type": "DATETIME",
"dest_name": "posteddt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "PTDUSRID",
"source_type": "CHAR(15)",
"dest_name": "ptdusrid",
"dest_type": "text",
"description": null
},
{
"source_name": "TEN99AMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "ten99amnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "RLGANLOS",
"source_type": "NUMERIC(19,5)",
"dest_name": "rlganlos",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "XCHGRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "xchgrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "DECPLCUR",
"source_type": "SMALLINT",
"dest_name": "decplcur",
"dest_type": "smallint",
"description": null
},
{
"source_name": "RATECALC",
"source_type": "SMALLINT",
"dest_name": "ratecalc",
"dest_type": "smallint",
"description": null
},
{
"source_name": "Payment_To_Print_On_Stub",
"source_type": "CHAR(21)",
"dest_name": "payment_to_print_on_stub",
"dest_type": "text",
"description": null
},
{
"source_name": "OAPYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "VADCDTRO",
"source_type": "CHAR(15)",
"dest_name": "vadcdtro",
"dest_type": "text",
"description": null
},
{
"source_name": "Credit1099Amount",
"source_type": "NUMERIC(19,5)",
"dest_name": "credit1099amount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DEFTEN99TYPE",
"source_type": "SMALLINT",
"dest_name": "deften99type",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEFTEN99BOXNUMBER",
"source_type": "SMALLINT",
"dest_name": "deften99boxnumber",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEX_ROW_ID",
"source_type": "INT",
"dest_name": "dex_row_id",
"dest_type": "integer",
"description": null
}
],
"watermarks": [
{
"name": "pm20100_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(LEAST(MAX(modifdt), CURRENT_DATE::timestamp) - INTERVAL '7 days', '1900-01-01') FROM gp.pm20100",
"default_value": "1900-01-01"
}
],
"hooks": [],
"description": "Dynamics GP open payables apply detail"
}
Reference in New Issue View Git Blame Copy Permalink