pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync the RM apply tables, rm30201 incrementally

Browse Source 
rm20201 (Applied Open) is 985 rows and stays a full reload. rm30201
(Apply History) was a full pull of 1,149,405 rows taking 19 minutes; it
now moves 661 rows in 6 s and reconcile --quick ties on all 108 metrics.

Neither apply table has DEX_ROW_TS, so rm30101's pattern does not port.
DEX_ROW_ID catches inserts but never moves on an update, so a 7-day
GLPOSTDT/APFRDCDT window carries restated and back-posted applies.
APTODCDT belongs to the apply-to document and lags by weeks; DATE1 has
163 rows dated to year 6201.

The pull is document-grain, not row-grain. merge_key is (aptodcty,
aptodcnm), so the DELETE clears every apply row for a document, and
staging only the row that matched would drop its siblings -- ~16k of
1.13M docs carry more than one. AK1/AK5 lead on those two columns, so
the EXISTS is an index seek.

Uses OPENQUERY pushdown rather than the four-part linked-server name: an
aggregate through the four-part name ran past 2 minutes against 1.3 s
pushed down. Note no apostrophes may appear inside the OPENQUERY string,
comments included.

Unapplies remain uncovered -- they delete the RM30201 row and write one
back to RM20201, and delete-by-key cannot see a row that is gone. The
985-row rm20201 reload is the cross-check.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-08-18 23:48:22 -04:00
parent 51ee71638f
commit 09056bea9d
4 changed files with 1029 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

435
config/modules/rm20201.json Normal file
View File

@ -0,0 +1,435 @@
{
"name": "rm20201",
"source_connection": "sqlserver",
"dest_connection": "usmidsap02",
"dest_table": "gp.rm20201",
"staging_table": "pipekit_staging.rm20201",
"merge_strategy": "full",
"merge_key": null,
"enabled": 1,
"dest_description": null,
"columns": [
{
"source_name": "CUSTNMBR",
"source_type": "CHAR(15)",
"dest_name": "custnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "CPRCSTNM",
"source_type": "CHAR(15)",
"dest_name": "cprcstnm",
"dest_type": "text",
"description": null
},
{
"source_name": "TRXSORCE",
"source_type": "CHAR(13)",
"dest_name": "trxsorce",
"dest_type": "text",
"description": null
},
{
"source_name": "DATE1",
"source_type": "DATETIME",
"dest_name": "date1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "TIME1",
"source_type": "DATETIME",
"dest_name": "time1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "GLPOSTDT",
"source_type": "DATETIME",
"dest_name": "glpostdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "POSTED",
"source_type": "TINYINT",
"dest_name": "posted",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TAXDTLID",
"source_type": "CHAR(15)",
"dest_name": "taxdtlid",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCNM",
"source_type": "CHAR(21)",
"dest_name": "aptodcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCTY",
"source_type": "SMALLINT",
"dest_name": "aptodcty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTODCDT",
"source_type": "DATETIME",
"dest_name": "aptodcdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ApplyToGLPostDate",
"source_type": "DATETIME",
"dest_name": "applytoglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "CURNCYID",
"source_type": "CHAR(15)",
"dest_name": "curncyid",
"dest_type": "text",
"description": null
},
{
"source_name": "CURRNIDX",
"source_type": "SMALLINT",
"dest_name": "currnidx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APPTOAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apptoamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISTKNAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "distknam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISAVTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "disavtkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "WROFAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "wrofamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORAPTOAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "oraptoam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDISTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordistkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDATKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordatkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORWROFAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "orwrofam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APTOEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptoexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTODENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptodenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTORTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "aptortclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTOMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "aptomctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRDCNM",
"source_type": "CHAR(21)",
"dest_name": "apfrdcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APFRDCTY",
"source_type": "SMALLINT",
"dest_name": "apfrdcty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRDCDT",
"source_type": "DATETIME",
"dest_name": "apfrdcdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ApplyFromGLPostDate",
"source_type": "DATETIME",
"dest_name": "applyfromglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "FROMCURR",
"source_type": "CHAR(15)",
"dest_name": "fromcurr",
"dest_type": "text",
"description": null
},
{
"source_name": "APFRMAPLYAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmaplyamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCTAKEN",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdisctaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCAVAIL",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdiscavail",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMWROFAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmwrofamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualApplyToAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualapplytoamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscTakenAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdisctakenamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscAvailTaken",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdiscavailtaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualWriteOffAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualwriteoffamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMDENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmdenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMRTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "apfrmrtclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRMMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "apfrmmctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "GSTDSAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "gstdsamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "PPSAMDED",
"source_type": "NUMERIC(19,5)",
"dest_name": "ppsamded",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "RLGANLOS",
"source_type": "NUMERIC(19,5)",
"dest_name": "rlganlos",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Revaluation_Status",
"source_type": "SMALLINT",
"dest_name": "revaluation_status",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEX_ROW_ID",
"source_type": "INT",
"dest_name": "dex_row_id",
"dest_type": "integer",
"description": null
}
],
"watermarks": [],
"hooks": []
}

62
config/modules/rm20201.sql Normal file
View File

@ -0,0 +1,62 @@
SELECT
RTRIM([CUSTNMBR]) AS [custnmbr],
RTRIM([CPRCSTNM]) AS [cprcstnm],
RTRIM([TRXSORCE]) AS [trxsorce],
[DATE1] AS [date1],
[TIME1] AS [time1],
[GLPOSTDT] AS [glpostdt],
[POSTED] AS [posted],
RTRIM([TAXDTLID]) AS [taxdtlid],
RTRIM([APTODCNM]) AS [aptodcnm],
[APTODCTY] AS [aptodcty],
[APTODCDT] AS [aptodcdt],
[ApplyToGLPostDate] AS [applytoglpostdate],
RTRIM([CURNCYID]) AS [curncyid],
[CURRNIDX] AS [currnidx],
[APPTOAMT] AS [apptoamt],
[DISTKNAM] AS [distknam],
[DISAVTKN] AS [disavtkn],
[WROFAMNT] AS [wrofamnt],
[ORAPTOAM] AS [oraptoam],
[ORDISTKN] AS [ordistkn],
[ORDATKN] AS [ordatkn],
[ORWROFAM] AS [orwrofam],
[APTOEXRATE] AS [aptoexrate],
[APTODENRATE] AS [aptodenrate],
[APTORTCLCMETH] AS [aptortclcmeth],
[APTOMCTRXSTT] AS [aptomctrxstt],
RTRIM([APFRDCNM]) AS [apfrdcnm],
[APFRDCTY] AS [apfrdcty],
[APFRDCDT] AS [apfrdcdt],
[ApplyFromGLPostDate] AS [applyfromglpostdate],
RTRIM([FROMCURR]) AS [fromcurr],
[APFRMAPLYAMT] AS [apfrmaplyamt],
[APFRMDISCTAKEN] AS [apfrmdisctaken],
[APFRMDISCAVAIL] AS [apfrmdiscavail],
[APFRMWROFAMT] AS [apfrmwrofamt],
[ActualApplyToAmount] AS [actualapplytoamount],
[ActualDiscTakenAmount] AS [actualdisctakenamount],
[ActualDiscAvailTaken] AS [actualdiscavailtaken],
[ActualWriteOffAmount] AS [actualwriteoffamount],
[APFRMEXRATE] AS [apfrmexrate],
[APFRMDENRATE] AS [apfrmdenrate],
[APFRMRTCLCMETH] AS [apfrmrtclcmeth],
[APFRMMCTRXSTT] AS [apfrmmctrxstt],
[APYFRMRNDAMT] AS [apyfrmrndamt],
[APYTORNDAMT] AS [apytorndamt],
[APYTORNDDISC] AS [apytornddisc],
[OAPYFRMRNDAMT] AS [oapyfrmrndamt],
[OAPYTORNDAMT] AS [oapytorndamt],
[OAPYTORNDDISC] AS [oapytornddisc],
[GSTDSAMT] AS [gstdsamt],
[PPSAMDED] AS [ppsamded],
[RLGANLOS] AS [rlganlos],
[Settled_Gain_CreditCurrT] AS [settled_gain_creditcurrt],
[Settled_Loss_CreditCurrT] AS [settled_loss_creditcurrt],
[Settled_Gain_DebitCurrTr] AS [settled_gain_debitcurrtr],
[Settled_Loss_DebitCurrTr] AS [settled_loss_debitcurrtr],
[Settled_Gain_DebitDiscAv] AS [settled_gain_debitdiscav],
[Settled_Loss_DebitDiscAv] AS [settled_loss_debitdiscav],
[Revaluation_Status] AS [revaluation_status],
[DEX_ROW_ID] AS [dex_row_id]
FROM [GPSERVER].[CHG].[dbo].[rm20201]

448
config/modules/rm30201.json Normal file
View File

@ -0,0 +1,448 @@
{
"name": "rm30201",
"source_connection": "sqlserver",
"dest_connection": "usmidsap02",
"dest_table": "gp.rm30201",
"staging_table": "pipekit_staging.rm30201",
"merge_strategy": "incremental",
"merge_key": "aptodcty, aptodcnm",
"enabled": 1,
"dest_description": null,
"columns": [
{
"source_name": "CUSTNMBR",
"source_type": "CHAR(15)",
"dest_name": "custnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "CPRCSTNM",
"source_type": "CHAR(15)",
"dest_name": "cprcstnm",
"dest_type": "text",
"description": null
},
{
"source_name": "TRXSORCE",
"source_type": "CHAR(13)",
"dest_name": "trxsorce",
"dest_type": "text",
"description": null
},
{
"source_name": "DATE1",
"source_type": "DATETIME",
"dest_name": "date1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "TIME1",
"source_type": "DATETIME",
"dest_name": "time1",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "GLPOSTDT",
"source_type": "DATETIME",
"dest_name": "glpostdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "POSTED",
"source_type": "TINYINT",
"dest_name": "posted",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TAXDTLID",
"source_type": "CHAR(15)",
"dest_name": "taxdtlid",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCNM",
"source_type": "CHAR(21)",
"dest_name": "aptodcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APTODCTY",
"source_type": "SMALLINT",
"dest_name": "aptodcty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTODCDT",
"source_type": "DATETIME",
"dest_name": "aptodcdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ApplyToGLPostDate",
"source_type": "DATETIME",
"dest_name": "applytoglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "CURNCYID",
"source_type": "CHAR(15)",
"dest_name": "curncyid",
"dest_type": "text",
"description": null
},
{
"source_name": "CURRNIDX",
"source_type": "SMALLINT",
"dest_name": "currnidx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APPTOAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apptoamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISTKNAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "distknam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISAVTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "disavtkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "WROFAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "wrofamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORAPTOAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "oraptoam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDISTKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordistkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDATKN",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordatkn",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORWROFAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "orwrofam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APTOEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptoexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTODENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "aptodenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APTORTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "aptortclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APTOMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "aptomctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRDCNM",
"source_type": "CHAR(21)",
"dest_name": "apfrdcnm",
"dest_type": "text",
"description": null
},
{
"source_name": "APFRDCTY",
"source_type": "SMALLINT",
"dest_name": "apfrdcty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRDCDT",
"source_type": "DATETIME",
"dest_name": "apfrdcdt",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ApplyFromGLPostDate",
"source_type": "DATETIME",
"dest_name": "applyfromglpostdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "FROMCURR",
"source_type": "CHAR(15)",
"dest_name": "fromcurr",
"dest_type": "text",
"description": null
},
{
"source_name": "APFRMAPLYAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmaplyamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCTAKEN",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdisctaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMDISCAVAIL",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmdiscavail",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMWROFAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apfrmwrofamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualApplyToAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualapplytoamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscTakenAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdisctakenamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualDiscAvailTaken",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualdiscavailtaken",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ActualWriteOffAmount",
"source_type": "NUMERIC(19,5)",
"dest_name": "actualwriteoffamount",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APFRMEXRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmexrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMDENRATE",
"source_type": "NUMERIC(19,7)",
"dest_name": "apfrmdenrate",
"dest_type": "numeric(19,7)",
"description": null
},
{
"source_name": "APFRMRTCLCMETH",
"source_type": "SMALLINT",
"dest_name": "apfrmrtclcmeth",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APFRMMCTRXSTT",
"source_type": "SMALLINT",
"dest_name": "apfrmmctrxstt",
"dest_type": "smallint",
"description": null
},
{
"source_name": "APYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "APYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "apytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYFRMRNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapyfrmrndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytorndamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OAPYTORNDDISC",
"source_type": "NUMERIC(19,5)",
"dest_name": "oapytornddisc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "GSTDSAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "gstdsamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "PPSAMDED",
"source_type": "NUMERIC(19,5)",
"dest_name": "ppsamded",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "RLGANLOS",
"source_type": "NUMERIC(19,5)",
"dest_name": "rlganlos",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_CreditCurrT",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_creditcurrt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitCurrTr",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitcurrtr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Gain_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_gain_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Settled_Loss_DebitDiscAv",
"source_type": "NUMERIC(19,5)",
"dest_name": "settled_loss_debitdiscav",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "Revaluation_Status",
"source_type": "SMALLINT",
"dest_name": "revaluation_status",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEX_ROW_ID",
"source_type": "INT",
"dest_name": "dex_row_id",
"dest_type": "integer",
"description": null
}
],
"watermarks": [
{
"name": "rm30201_dt",
"connection": "usmidsap02",
"resolver_sql": "SELECT TO_CHAR(LEAST(COALESCE(MAX(glpostdt)::DATE, CURRENT_DATE), CURRENT_DATE) - 7, 'YYYY-MM-DD') FROM gp.rm30201",
"default_value": null
},
{
"name": "rm30201_id",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_id), 0) FROM gp.rm30201",
"default_value": null
}
],
"hooks": []
}

84
config/modules/rm30201.sql Normal file
View File

@ -0,0 +1,84 @@
SELECT * FROM OPENQUERY(GPSERVER, '
SELECT
RTRIM([CUSTNMBR]) AS [custnmbr],
RTRIM([CPRCSTNM]) AS [cprcstnm],
RTRIM([TRXSORCE]) AS [trxsorce],
[DATE1] AS [date1],
[TIME1] AS [time1],
[GLPOSTDT] AS [glpostdt],
[POSTED] AS [posted],
RTRIM([TAXDTLID]) AS [taxdtlid],
RTRIM([APTODCNM]) AS [aptodcnm],
[APTODCTY] AS [aptodcty],
[APTODCDT] AS [aptodcdt],
[ApplyToGLPostDate] AS [applytoglpostdate],
RTRIM([CURNCYID]) AS [curncyid],
[CURRNIDX] AS [currnidx],
[APPTOAMT] AS [apptoamt],
[DISTKNAM] AS [distknam],
[DISAVTKN] AS [disavtkn],
[WROFAMNT] AS [wrofamnt],
[ORAPTOAM] AS [oraptoam],
[ORDISTKN] AS [ordistkn],
[ORDATKN] AS [ordatkn],
[ORWROFAM] AS [orwrofam],
[APTOEXRATE] AS [aptoexrate],
[APTODENRATE] AS [aptodenrate],
[APTORTCLCMETH] AS [aptortclcmeth],
[APTOMCTRXSTT] AS [aptomctrxstt],
RTRIM([APFRDCNM]) AS [apfrdcnm],
[APFRDCTY] AS [apfrdcty],
[APFRDCDT] AS [apfrdcdt],
[ApplyFromGLPostDate] AS [applyfromglpostdate],
RTRIM([FROMCURR]) AS [fromcurr],
[APFRMAPLYAMT] AS [apfrmaplyamt],
[APFRMDISCTAKEN] AS [apfrmdisctaken],
[APFRMDISCAVAIL] AS [apfrmdiscavail],
[APFRMWROFAMT] AS [apfrmwrofamt],
[ActualApplyToAmount] AS [actualapplytoamount],
[ActualDiscTakenAmount] AS [actualdisctakenamount],
[ActualDiscAvailTaken] AS [actualdiscavailtaken],
[ActualWriteOffAmount] AS [actualwriteoffamount],
[APFRMEXRATE] AS [apfrmexrate],
[APFRMDENRATE] AS [apfrmdenrate],
[APFRMRTCLCMETH] AS [apfrmrtclcmeth],
[APFRMMCTRXSTT] AS [apfrmmctrxstt],
[APYFRMRNDAMT] AS [apyfrmrndamt],
[APYTORNDAMT] AS [apytorndamt],
[APYTORNDDISC] AS [apytornddisc],
[OAPYFRMRNDAMT] AS [oapyfrmrndamt],
[OAPYTORNDAMT] AS [oapytorndamt],
[OAPYTORNDDISC] AS [oapytornddisc],
[GSTDSAMT] AS [gstdsamt],
[PPSAMDED] AS [ppsamded],
[RLGANLOS] AS [rlganlos],
[Settled_Gain_CreditCurrT] AS [settled_gain_creditcurrt],
[Settled_Loss_CreditCurrT] AS [settled_loss_creditcurrt],
[Settled_Gain_DebitCurrTr] AS [settled_gain_debitcurrtr],
[Settled_Loss_DebitCurrTr] AS [settled_loss_debitcurrtr],
[Settled_Gain_DebitDiscAv] AS [settled_gain_debitdiscav],
[Settled_Loss_DebitDiscAv] AS [settled_loss_debitdiscav],
[Revaluation_Status] AS [revaluation_status],
[DEX_ROW_ID] AS [dex_row_id]
FROM CHG.dbo.RM30201 t
-- RM30201 has no DEX_ROW_TS (only the RM*0101 pair carries one), so there is
-- no single change stamp: DEX_ROW_ID catches inserts but never moves on an
-- update, and the GLPOSTDT / APFRDCDT window catches restated or back-posted
-- applies. APTODCDT belongs to the apply-TO document, so it lags by weeks and
-- is not a driver; DATE1 has 163 rows dated out to year 6201 and is unusable.
--
-- merge_key is (aptodcty, aptodcnm), so the DELETE clears every apply row for a
-- document -- the pull must therefore be document-grain, not row-grain, or a
-- doc holding two apply rows would lose the sibling the predicate missed.
-- ~16k of 1.13M docs carry more than one row. AK1/AK5 lead on
-- APTODCTY, APTODCNM, so the EXISTS is an index seek.
WHERE EXISTS (
SELECT 1
FROM CHG.dbo.RM30201 c
WHERE c.[APTODCTY] = t.[APTODCTY]
AND c.[APTODCNM] = t.[APTODCNM]
AND ( c.[DEX_ROW_ID] > {rm30201_id}
OR c.[GLPOSTDT] >= ''{rm30201_dt}''
OR c.[APFRDCDT] >= ''{rm30201_dt}'')
)
')