From 09056bea9d4b3ffa16345ca5fd038cec3f6f35f6 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 18 Aug 2026 23:48:22 -0400 Subject: [PATCH] Sync the RM apply tables, rm30201 incrementally rm20201 (Applied Open) is 985 rows and stays a full reload. rm30201 (Apply History) was a full pull of 1,149,405 rows taking 19 minutes; it now moves 661 rows in 6 s and reconcile --quick ties on all 108 metrics. Neither apply table has DEX_ROW_TS, so rm30101's pattern does not port. DEX_ROW_ID catches inserts but never moves on an update, so a 7-day GLPOSTDT/APFRDCDT window carries restated and back-posted applies. APTODCDT belongs to the apply-to document and lags by weeks; DATE1 has 163 rows dated to year 6201. The pull is document-grain, not row-grain. merge_key is (aptodcty, aptodcnm), so the DELETE clears every apply row for a document, and staging only the row that matched would drop its siblings -- ~16k of 1.13M docs carry more than one. AK1/AK5 lead on those two columns, so the EXISTS is an index seek. Uses OPENQUERY pushdown rather than the four-part linked-server name: an aggregate through the four-part name ran past 2 minutes against 1.3 s pushed down. Note no apostrophes may appear inside the OPENQUERY string, comments included. Unapplies remain uncovered -- they delete the RM30201 row and write one back to RM20201, and delete-by-key cannot see a row that is gone. The 985-row rm20201 reload is the cross-check. Co-Authored-By: Claude Opus 5 --- config/modules/rm20201.json | 435 ++++++++++++++++++++++++++++++++++ config/modules/rm20201.sql | 62 +++++ config/modules/rm30201.json | 448 ++++++++++++++++++++++++++++++++++++ config/modules/rm30201.sql | 84 +++++++ 4 files changed, 1029 insertions(+) create mode 100644 config/modules/rm20201.json create mode 100644 config/modules/rm20201.sql create mode 100644 config/modules/rm30201.json create mode 100644 config/modules/rm30201.sql diff --git a/config/modules/rm20201.json b/config/modules/rm20201.json new file mode 100644 index 0000000..d2a344d --- /dev/null +++ b/config/modules/rm20201.json @@ -0,0 +1,435 @@ +{ + "name": "rm20201", + "source_connection": "sqlserver", + "dest_connection": "usmidsap02", + "dest_table": "gp.rm20201", + "staging_table": "pipekit_staging.rm20201", + "merge_strategy": "full", + "merge_key": null, + "enabled": 1, + "dest_description": null, + "columns": [ + { + "source_name": "CUSTNMBR", + "source_type": "CHAR(15)", + "dest_name": "custnmbr", + "dest_type": "text", + "description": null + }, + { + "source_name": "CPRCSTNM", + "source_type": "CHAR(15)", + "dest_name": "cprcstnm", + "dest_type": "text", + "description": null + }, + { + "source_name": "TRXSORCE", + "source_type": "CHAR(13)", + "dest_name": "trxsorce", + "dest_type": "text", + "description": null + }, + { + "source_name": "DATE1", + "source_type": "DATETIME", + "dest_name": "date1", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "TIME1", + "source_type": "DATETIME", + "dest_name": "time1", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "GLPOSTDT", + "source_type": "DATETIME", + "dest_name": "glpostdt", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "POSTED", + "source_type": "TINYINT", + "dest_name": "posted", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "TAXDTLID", + "source_type": "CHAR(15)", + "dest_name": "taxdtlid", + "dest_type": "text", + "description": null + }, + { + "source_name": "APTODCNM", + "source_type": "CHAR(21)", + "dest_name": "aptodcnm", + "dest_type": "text", + "description": null + }, + { + "source_name": "APTODCTY", + "source_type": "SMALLINT", + "dest_name": "aptodcty", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APTODCDT", + "source_type": "DATETIME", + "dest_name": "aptodcdt", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "ApplyToGLPostDate", + "source_type": "DATETIME", + "dest_name": "applytoglpostdate", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "CURNCYID", + "source_type": "CHAR(15)", + "dest_name": "curncyid", + "dest_type": "text", + "description": null + }, + { + "source_name": "CURRNIDX", + "source_type": "SMALLINT", + "dest_name": "currnidx", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APPTOAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apptoamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "DISTKNAM", + "source_type": "NUMERIC(19,5)", + "dest_name": "distknam", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "DISAVTKN", + "source_type": "NUMERIC(19,5)", + "dest_name": "disavtkn", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "WROFAMNT", + "source_type": "NUMERIC(19,5)", + "dest_name": "wrofamnt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORAPTOAM", + "source_type": "NUMERIC(19,5)", + "dest_name": "oraptoam", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORDISTKN", + "source_type": "NUMERIC(19,5)", + "dest_name": "ordistkn", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORDATKN", + "source_type": "NUMERIC(19,5)", + "dest_name": "ordatkn", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORWROFAM", + "source_type": "NUMERIC(19,5)", + "dest_name": "orwrofam", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APTOEXRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "aptoexrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APTODENRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "aptodenrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APTORTCLCMETH", + "source_type": "SMALLINT", + "dest_name": "aptortclcmeth", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APTOMCTRXSTT", + "source_type": "SMALLINT", + "dest_name": "aptomctrxstt", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APFRDCNM", + "source_type": "CHAR(21)", + "dest_name": "apfrdcnm", + "dest_type": "text", + "description": null + }, + { + "source_name": "APFRDCTY", + "source_type": "SMALLINT", + "dest_name": "apfrdcty", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APFRDCDT", + "source_type": "DATETIME", + "dest_name": "apfrdcdt", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "ApplyFromGLPostDate", + "source_type": "DATETIME", + "dest_name": "applyfromglpostdate", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "FROMCURR", + "source_type": "CHAR(15)", + "dest_name": "fromcurr", + "dest_type": "text", + "description": null + }, + { + "source_name": "APFRMAPLYAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmaplyamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMDISCTAKEN", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmdisctaken", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMDISCAVAIL", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmdiscavail", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMWROFAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmwrofamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualApplyToAmount", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualapplytoamount", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualDiscTakenAmount", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualdisctakenamount", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualDiscAvailTaken", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualdiscavailtaken", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualWriteOffAmount", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualwriteoffamount", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMEXRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "apfrmexrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APFRMDENRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "apfrmdenrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APFRMRTCLCMETH", + "source_type": "SMALLINT", + "dest_name": "apfrmrtclcmeth", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APFRMMCTRXSTT", + "source_type": "SMALLINT", + "dest_name": "apfrmmctrxstt", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APYFRMRNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apyfrmrndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APYTORNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apytorndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APYTORNDDISC", + "source_type": "NUMERIC(19,5)", + "dest_name": "apytornddisc", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "OAPYFRMRNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "oapyfrmrndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "OAPYTORNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "oapytorndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "OAPYTORNDDISC", + "source_type": "NUMERIC(19,5)", + "dest_name": "oapytornddisc", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "GSTDSAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "gstdsamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "PPSAMDED", + "source_type": "NUMERIC(19,5)", + "dest_name": "ppsamded", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "RLGANLOS", + "source_type": "NUMERIC(19,5)", + "dest_name": "rlganlos", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Gain_CreditCurrT", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_gain_creditcurrt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Loss_CreditCurrT", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_loss_creditcurrt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Gain_DebitCurrTr", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_gain_debitcurrtr", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Loss_DebitCurrTr", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_loss_debitcurrtr", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Gain_DebitDiscAv", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_gain_debitdiscav", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Loss_DebitDiscAv", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_loss_debitdiscav", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Revaluation_Status", + "source_type": "SMALLINT", + "dest_name": "revaluation_status", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "DEX_ROW_ID", + "source_type": "INT", + "dest_name": "dex_row_id", + "dest_type": "integer", + "description": null + } + ], + "watermarks": [], + "hooks": [] +} diff --git a/config/modules/rm20201.sql b/config/modules/rm20201.sql new file mode 100644 index 0000000..7e1d637 --- /dev/null +++ b/config/modules/rm20201.sql @@ -0,0 +1,62 @@ +SELECT + RTRIM([CUSTNMBR]) AS [custnmbr], + RTRIM([CPRCSTNM]) AS [cprcstnm], + RTRIM([TRXSORCE]) AS [trxsorce], + [DATE1] AS [date1], + [TIME1] AS [time1], + [GLPOSTDT] AS [glpostdt], + [POSTED] AS [posted], + RTRIM([TAXDTLID]) AS [taxdtlid], + RTRIM([APTODCNM]) AS [aptodcnm], + [APTODCTY] AS [aptodcty], + [APTODCDT] AS [aptodcdt], + [ApplyToGLPostDate] AS [applytoglpostdate], + RTRIM([CURNCYID]) AS [curncyid], + [CURRNIDX] AS [currnidx], + [APPTOAMT] AS [apptoamt], + [DISTKNAM] AS [distknam], + [DISAVTKN] AS [disavtkn], + [WROFAMNT] AS [wrofamnt], + [ORAPTOAM] AS [oraptoam], + [ORDISTKN] AS [ordistkn], + [ORDATKN] AS [ordatkn], + [ORWROFAM] AS [orwrofam], + [APTOEXRATE] AS [aptoexrate], + [APTODENRATE] AS [aptodenrate], + [APTORTCLCMETH] AS [aptortclcmeth], + [APTOMCTRXSTT] AS [aptomctrxstt], + RTRIM([APFRDCNM]) AS [apfrdcnm], + [APFRDCTY] AS [apfrdcty], + [APFRDCDT] AS [apfrdcdt], + [ApplyFromGLPostDate] AS [applyfromglpostdate], + RTRIM([FROMCURR]) AS [fromcurr], + [APFRMAPLYAMT] AS [apfrmaplyamt], + [APFRMDISCTAKEN] AS [apfrmdisctaken], + [APFRMDISCAVAIL] AS [apfrmdiscavail], + [APFRMWROFAMT] AS [apfrmwrofamt], + [ActualApplyToAmount] AS [actualapplytoamount], + [ActualDiscTakenAmount] AS [actualdisctakenamount], + [ActualDiscAvailTaken] AS [actualdiscavailtaken], + [ActualWriteOffAmount] AS [actualwriteoffamount], + [APFRMEXRATE] AS [apfrmexrate], + [APFRMDENRATE] AS [apfrmdenrate], + [APFRMRTCLCMETH] AS [apfrmrtclcmeth], + [APFRMMCTRXSTT] AS [apfrmmctrxstt], + [APYFRMRNDAMT] AS [apyfrmrndamt], + [APYTORNDAMT] AS [apytorndamt], + [APYTORNDDISC] AS [apytornddisc], + [OAPYFRMRNDAMT] AS [oapyfrmrndamt], + [OAPYTORNDAMT] AS [oapytorndamt], + [OAPYTORNDDISC] AS [oapytornddisc], + [GSTDSAMT] AS [gstdsamt], + [PPSAMDED] AS [ppsamded], + [RLGANLOS] AS [rlganlos], + [Settled_Gain_CreditCurrT] AS [settled_gain_creditcurrt], + [Settled_Loss_CreditCurrT] AS [settled_loss_creditcurrt], + [Settled_Gain_DebitCurrTr] AS [settled_gain_debitcurrtr], + [Settled_Loss_DebitCurrTr] AS [settled_loss_debitcurrtr], + [Settled_Gain_DebitDiscAv] AS [settled_gain_debitdiscav], + [Settled_Loss_DebitDiscAv] AS [settled_loss_debitdiscav], + [Revaluation_Status] AS [revaluation_status], + [DEX_ROW_ID] AS [dex_row_id] +FROM [GPSERVER].[CHG].[dbo].[rm20201] diff --git a/config/modules/rm30201.json b/config/modules/rm30201.json new file mode 100644 index 0000000..eb7095d --- /dev/null +++ b/config/modules/rm30201.json @@ -0,0 +1,448 @@ +{ + "name": "rm30201", + "source_connection": "sqlserver", + "dest_connection": "usmidsap02", + "dest_table": "gp.rm30201", + "staging_table": "pipekit_staging.rm30201", + "merge_strategy": "incremental", + "merge_key": "aptodcty, aptodcnm", + "enabled": 1, + "dest_description": null, + "columns": [ + { + "source_name": "CUSTNMBR", + "source_type": "CHAR(15)", + "dest_name": "custnmbr", + "dest_type": "text", + "description": null + }, + { + "source_name": "CPRCSTNM", + "source_type": "CHAR(15)", + "dest_name": "cprcstnm", + "dest_type": "text", + "description": null + }, + { + "source_name": "TRXSORCE", + "source_type": "CHAR(13)", + "dest_name": "trxsorce", + "dest_type": "text", + "description": null + }, + { + "source_name": "DATE1", + "source_type": "DATETIME", + "dest_name": "date1", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "TIME1", + "source_type": "DATETIME", + "dest_name": "time1", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "GLPOSTDT", + "source_type": "DATETIME", + "dest_name": "glpostdt", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "POSTED", + "source_type": "TINYINT", + "dest_name": "posted", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "TAXDTLID", + "source_type": "CHAR(15)", + "dest_name": "taxdtlid", + "dest_type": "text", + "description": null + }, + { + "source_name": "APTODCNM", + "source_type": "CHAR(21)", + "dest_name": "aptodcnm", + "dest_type": "text", + "description": null + }, + { + "source_name": "APTODCTY", + "source_type": "SMALLINT", + "dest_name": "aptodcty", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APTODCDT", + "source_type": "DATETIME", + "dest_name": "aptodcdt", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "ApplyToGLPostDate", + "source_type": "DATETIME", + "dest_name": "applytoglpostdate", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "CURNCYID", + "source_type": "CHAR(15)", + "dest_name": "curncyid", + "dest_type": "text", + "description": null + }, + { + "source_name": "CURRNIDX", + "source_type": "SMALLINT", + "dest_name": "currnidx", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APPTOAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apptoamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "DISTKNAM", + "source_type": "NUMERIC(19,5)", + "dest_name": "distknam", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "DISAVTKN", + "source_type": "NUMERIC(19,5)", + "dest_name": "disavtkn", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "WROFAMNT", + "source_type": "NUMERIC(19,5)", + "dest_name": "wrofamnt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORAPTOAM", + "source_type": "NUMERIC(19,5)", + "dest_name": "oraptoam", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORDISTKN", + "source_type": "NUMERIC(19,5)", + "dest_name": "ordistkn", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORDATKN", + "source_type": "NUMERIC(19,5)", + "dest_name": "ordatkn", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ORWROFAM", + "source_type": "NUMERIC(19,5)", + "dest_name": "orwrofam", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APTOEXRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "aptoexrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APTODENRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "aptodenrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APTORTCLCMETH", + "source_type": "SMALLINT", + "dest_name": "aptortclcmeth", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APTOMCTRXSTT", + "source_type": "SMALLINT", + "dest_name": "aptomctrxstt", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APFRDCNM", + "source_type": "CHAR(21)", + "dest_name": "apfrdcnm", + "dest_type": "text", + "description": null + }, + { + "source_name": "APFRDCTY", + "source_type": "SMALLINT", + "dest_name": "apfrdcty", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APFRDCDT", + "source_type": "DATETIME", + "dest_name": "apfrdcdt", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "ApplyFromGLPostDate", + "source_type": "DATETIME", + "dest_name": "applyfromglpostdate", + "dest_type": "timestamp", + "description": null + }, + { + "source_name": "FROMCURR", + "source_type": "CHAR(15)", + "dest_name": "fromcurr", + "dest_type": "text", + "description": null + }, + { + "source_name": "APFRMAPLYAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmaplyamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMDISCTAKEN", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmdisctaken", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMDISCAVAIL", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmdiscavail", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMWROFAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apfrmwrofamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualApplyToAmount", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualapplytoamount", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualDiscTakenAmount", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualdisctakenamount", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualDiscAvailTaken", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualdiscavailtaken", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "ActualWriteOffAmount", + "source_type": "NUMERIC(19,5)", + "dest_name": "actualwriteoffamount", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APFRMEXRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "apfrmexrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APFRMDENRATE", + "source_type": "NUMERIC(19,7)", + "dest_name": "apfrmdenrate", + "dest_type": "numeric(19,7)", + "description": null + }, + { + "source_name": "APFRMRTCLCMETH", + "source_type": "SMALLINT", + "dest_name": "apfrmrtclcmeth", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APFRMMCTRXSTT", + "source_type": "SMALLINT", + "dest_name": "apfrmmctrxstt", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "APYFRMRNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apyfrmrndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APYTORNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "apytorndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "APYTORNDDISC", + "source_type": "NUMERIC(19,5)", + "dest_name": "apytornddisc", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "OAPYFRMRNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "oapyfrmrndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "OAPYTORNDAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "oapytorndamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "OAPYTORNDDISC", + "source_type": "NUMERIC(19,5)", + "dest_name": "oapytornddisc", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "GSTDSAMT", + "source_type": "NUMERIC(19,5)", + "dest_name": "gstdsamt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "PPSAMDED", + "source_type": "NUMERIC(19,5)", + "dest_name": "ppsamded", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "RLGANLOS", + "source_type": "NUMERIC(19,5)", + "dest_name": "rlganlos", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Gain_CreditCurrT", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_gain_creditcurrt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Loss_CreditCurrT", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_loss_creditcurrt", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Gain_DebitCurrTr", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_gain_debitcurrtr", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Loss_DebitCurrTr", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_loss_debitcurrtr", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Gain_DebitDiscAv", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_gain_debitdiscav", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Settled_Loss_DebitDiscAv", + "source_type": "NUMERIC(19,5)", + "dest_name": "settled_loss_debitdiscav", + "dest_type": "numeric(19,5)", + "description": null + }, + { + "source_name": "Revaluation_Status", + "source_type": "SMALLINT", + "dest_name": "revaluation_status", + "dest_type": "smallint", + "description": null + }, + { + "source_name": "DEX_ROW_ID", + "source_type": "INT", + "dest_name": "dex_row_id", + "dest_type": "integer", + "description": null + } + ], + "watermarks": [ + { + "name": "rm30201_dt", + "connection": "usmidsap02", + "resolver_sql": "SELECT TO_CHAR(LEAST(COALESCE(MAX(glpostdt)::DATE, CURRENT_DATE), CURRENT_DATE) - 7, 'YYYY-MM-DD') FROM gp.rm30201", + "default_value": null + }, + { + "name": "rm30201_id", + "connection": "usmidsap02", + "resolver_sql": "SELECT COALESCE(MAX(dex_row_id), 0) FROM gp.rm30201", + "default_value": null + } + ], + "hooks": [] +} diff --git a/config/modules/rm30201.sql b/config/modules/rm30201.sql new file mode 100644 index 0000000..c32979f --- /dev/null +++ b/config/modules/rm30201.sql @@ -0,0 +1,84 @@ +SELECT * FROM OPENQUERY(GPSERVER, ' +SELECT + RTRIM([CUSTNMBR]) AS [custnmbr], + RTRIM([CPRCSTNM]) AS [cprcstnm], + RTRIM([TRXSORCE]) AS [trxsorce], + [DATE1] AS [date1], + [TIME1] AS [time1], + [GLPOSTDT] AS [glpostdt], + [POSTED] AS [posted], + RTRIM([TAXDTLID]) AS [taxdtlid], + RTRIM([APTODCNM]) AS [aptodcnm], + [APTODCTY] AS [aptodcty], + [APTODCDT] AS [aptodcdt], + [ApplyToGLPostDate] AS [applytoglpostdate], + RTRIM([CURNCYID]) AS [curncyid], + [CURRNIDX] AS [currnidx], + [APPTOAMT] AS [apptoamt], + [DISTKNAM] AS [distknam], + [DISAVTKN] AS [disavtkn], + [WROFAMNT] AS [wrofamnt], + [ORAPTOAM] AS [oraptoam], + [ORDISTKN] AS [ordistkn], + [ORDATKN] AS [ordatkn], + [ORWROFAM] AS [orwrofam], + [APTOEXRATE] AS [aptoexrate], + [APTODENRATE] AS [aptodenrate], + [APTORTCLCMETH] AS [aptortclcmeth], + [APTOMCTRXSTT] AS [aptomctrxstt], + RTRIM([APFRDCNM]) AS [apfrdcnm], + [APFRDCTY] AS [apfrdcty], + [APFRDCDT] AS [apfrdcdt], + [ApplyFromGLPostDate] AS [applyfromglpostdate], + RTRIM([FROMCURR]) AS [fromcurr], + [APFRMAPLYAMT] AS [apfrmaplyamt], + [APFRMDISCTAKEN] AS [apfrmdisctaken], + [APFRMDISCAVAIL] AS [apfrmdiscavail], + [APFRMWROFAMT] AS [apfrmwrofamt], + [ActualApplyToAmount] AS [actualapplytoamount], + [ActualDiscTakenAmount] AS [actualdisctakenamount], + [ActualDiscAvailTaken] AS [actualdiscavailtaken], + [ActualWriteOffAmount] AS [actualwriteoffamount], + [APFRMEXRATE] AS [apfrmexrate], + [APFRMDENRATE] AS [apfrmdenrate], + [APFRMRTCLCMETH] AS [apfrmrtclcmeth], + [APFRMMCTRXSTT] AS [apfrmmctrxstt], + [APYFRMRNDAMT] AS [apyfrmrndamt], + [APYTORNDAMT] AS [apytorndamt], + [APYTORNDDISC] AS [apytornddisc], + [OAPYFRMRNDAMT] AS [oapyfrmrndamt], + [OAPYTORNDAMT] AS [oapytorndamt], + [OAPYTORNDDISC] AS [oapytornddisc], + [GSTDSAMT] AS [gstdsamt], + [PPSAMDED] AS [ppsamded], + [RLGANLOS] AS [rlganlos], + [Settled_Gain_CreditCurrT] AS [settled_gain_creditcurrt], + [Settled_Loss_CreditCurrT] AS [settled_loss_creditcurrt], + [Settled_Gain_DebitCurrTr] AS [settled_gain_debitcurrtr], + [Settled_Loss_DebitCurrTr] AS [settled_loss_debitcurrtr], + [Settled_Gain_DebitDiscAv] AS [settled_gain_debitdiscav], + [Settled_Loss_DebitDiscAv] AS [settled_loss_debitdiscav], + [Revaluation_Status] AS [revaluation_status], + [DEX_ROW_ID] AS [dex_row_id] +FROM CHG.dbo.RM30201 t +-- RM30201 has no DEX_ROW_TS (only the RM*0101 pair carries one), so there is +-- no single change stamp: DEX_ROW_ID catches inserts but never moves on an +-- update, and the GLPOSTDT / APFRDCDT window catches restated or back-posted +-- applies. APTODCDT belongs to the apply-TO document, so it lags by weeks and +-- is not a driver; DATE1 has 163 rows dated out to year 6201 and is unusable. +-- +-- merge_key is (aptodcty, aptodcnm), so the DELETE clears every apply row for a +-- document -- the pull must therefore be document-grain, not row-grain, or a +-- doc holding two apply rows would lose the sibling the predicate missed. +-- ~16k of 1.13M docs carry more than one row. AK1/AK5 lead on +-- APTODCTY, APTODCNM, so the EXISTS is an index seek. +WHERE EXISTS ( + SELECT 1 + FROM CHG.dbo.RM30201 c + WHERE c.[APTODCTY] = t.[APTODCTY] + AND c.[APTODCNM] = t.[APTODCNM] + AND ( c.[DEX_ROW_ID] > {rm30201_id} + OR c.[GLPOSTDT] >= ''{rm30201_dt}'' + OR c.[APFRDCDT] >= ''{rm30201_dt}'') +) +')