pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
890f10cbab
pipekit/config/modules/gtran.json
Paul Trowbridge dd9f89f4b5 feat: version-control pipeline definitions via export/apply 
Add `pipekit export` / `pipekit apply` (pipekit/config_io.py) to serialise the
DB's config — drivers, connections, modules (+ columns, watermarks, hooks),
groups, schedules — to a git-trackable text tree under config/, and rehydrate
it. SQLite stays runtime state; definitions become diffable/reviewable/
revertible.

- config only: run_log/group_run/settings and per-run state columns excluded
- name-keyed refs (portable across databases); source_query in .sql sidecars;
  columns as real JSON arrays for line-by-line diffs
- newline-normalised so CRLF-vs-LF is never a spurious change
- apply is create/update by name; child collections fully synced; top-level
  deletes gated behind --prune; --dry-run prints the plan
- round-trip is identity (export -> apply --dry-run == nothing to do)

Commits the current config/ as the first baseline, capturing the freshly
populated columns_json for rm00101/rm00301/iv00101. Documented in SPEC.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 10:56:55 -04:00

646 lines
18 KiB
JSON
Raw Blame History

{
"name": "gtran",
"source_connection": "S78030956",
"dest_connection": "usmidsap02",
"dest_table": "cms.gtran",
"staging_table": "pipekit_staging.gtran",
"merge_strategy": "append",
"merge_key": null,
"enabled": 1,
"dest_description": "G/L Transactions 6.1",
"columns": [
{
"source_name": "DKACC#",
"source_type": "NUMERIC(12,0)",
"dest_name": "dkacc#",
"dest_type": "numeric(12,0)",
"description": "G/L Account Number with Company"
},
{
"source_name": "DKFSPR",
"source_type": "NUMERIC(2,0)",
"dest_name": "dkfspr",
"dest_type": "numeric(2,0)",
"description": "Fiscal Period"
},
{
"source_name": "DKJRTP",
"source_type": "CHAR(1)",
"dest_name": "dkjrtp",
"dest_type": "text",
"description": "Journal Type"
},
{
"source_name": "DKTDAT",
"source_type": "DATE",
"dest_name": "dktdat",
"dest_type": "date",
"description": "Transaction Date"
},
{
"source_name": "DKSRCE",
"source_type": "CHAR(2)",
"dest_name": "dksrce",
"dest_type": "text",
"description": "Transaction Source"
},
{
"source_name": "DKAMT",
"source_type": "NUMERIC(14,2)",
"dest_name": "dkamt",
"dest_type": "numeric(14,2)",
"description": "Transaction Amount"
},
{
"source_name": "DKREF#",
"source_type": "CHAR(10)",
"dest_name": "dkref#",
"dest_type": "text",
"description": "Reference Number"
},
{
"source_name": "DKREFD",
"source_type": "CHAR(30)",
"dest_name": "dkrefd",
"dest_type": "text",
"description": "Reference Description"
},
{
"source_name": "DKADDD",
"source_type": "CHAR(30)",
"dest_name": "dkaddd",
"dest_type": "text",
"description": "Additional Description"
},
{
"source_name": "DKREV",
"source_type": "CHAR(1)",
"dest_name": "dkrev",
"dest_type": "text",
"description": "Reversing Entry"
},
{
"source_name": "DKREVS",
"source_type": "CHAR(1)",
"dest_name": "dkrevs",
"dest_type": "text",
"description": "Reversing Status"
},
{
"source_name": "DKFSYR",
"source_type": "NUMERIC(2,0)",
"dest_name": "dkfsyr",
"dest_type": "numeric(2,0)",
"description": "Fiscal Century"
},
{
"source_name": "DKFSYY",
"source_type": "NUMERIC(2,0)",
"dest_name": "dkfsyy",
"dest_type": "numeric(2,0)",
"description": "Fiscal Year YY"
},
{
"source_name": "DKPOST",
"source_type": "CHAR(1)",
"dest_name": "dkpost",
"dest_type": "text",
"description": "Posted Entry Y/N"
},
{
"source_name": "DKPDAT",
"source_type": "DATE",
"dest_name": "dkpdat",
"dest_type": "date",
"description": "Posted Date"
},
{
"source_name": "DKBTC#",
"source_type": "NUMERIC(9,0)",
"dest_name": "dkbtc#",
"dest_type": "numeric(9,0)",
"description": "Batch Number"
},
{
"source_name": "DKQUAL",
"source_type": "CHAR(2)",
"dest_name": "dkqual",
"dest_type": "text",
"description": "Key Qualifier"
},
{
"source_name": "DKKEYN",
"source_type": "CHAR(20)",
"dest_name": "dkkeyn",
"dest_type": "text",
"description": "Source Key"
},
{
"source_name": "DKPART",
"source_type": "CHAR(20)",
"dest_name": "dkpart",
"dest_type": "text",
"description": "Part Number"
},
{
"source_name": "DKPJNM",
"source_type": "CHAR(20)",
"dest_name": "dkpjnm",
"dest_type": "text",
"description": "Project Number"
},
{
"source_name": "DKQTY",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkqty",
"dest_type": "numeric(15,5)",
"description": "Quantity"
},
{
"source_name": "DKUNIT",
"source_type": "CHAR(3)",
"dest_name": "dkunit",
"dest_type": "text",
"description": "Unit of Measure"
},
{
"source_name": "DKFUT1",
"source_type": "CHAR(1)",
"dest_name": "dkfut1",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT2",
"source_type": "CHAR(1)",
"dest_name": "dkfut2",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT3",
"source_type": "CHAR(1)",
"dest_name": "dkfut3",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT4",
"source_type": "CHAR(10)",
"dest_name": "dkfut4",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT5",
"source_type": "CHAR(10)",
"dest_name": "dkfut5",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT6",
"source_type": "CHAR(20)",
"dest_name": "dkfut6",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT7",
"source_type": "NUMERIC(5,0)",
"dest_name": "dkfut7",
"dest_type": "numeric(5,0)",
"description": "Future Use"
},
{
"source_name": "DKFUT8",
"source_type": "CHAR(3)",
"dest_name": "dkfut8",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT9",
"source_type": "CHAR(3)",
"dest_name": "dkfut9",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT10",
"source_type": "CHAR(5)",
"dest_name": "dkfut10",
"dest_type": "text",
"description": "Journal Line #"
},
{
"source_name": "DKFUT11",
"source_type": "CHAR(5)",
"dest_name": "dkfut11",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT12",
"source_type": "CHAR(10)",
"dest_name": "dkfut12",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT13",
"source_type": "CHAR(10)",
"dest_name": "dkfut13",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT14",
"source_type": "NUMERIC(11,2)",
"dest_name": "dkfut14",
"dest_type": "numeric(11,2)",
"description": "Future Use"
},
{
"source_name": "DKFUT15",
"source_type": "NUMERIC(11,2)",
"dest_name": "dkfut15",
"dest_type": "numeric(11,2)",
"description": "Future Use"
},
{
"source_name": "DKFUT16",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkfut16",
"dest_type": "numeric(15,5)",
"description": "Future Use"
},
{
"source_name": "DKFUT17",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkfut17",
"dest_type": "numeric(15,5)",
"description": "Future Use"
},
{
"source_name": "DKFUT18",
"source_type": "CHAR(1)",
"dest_name": "dkfut18",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT19",
"source_type": "CHAR(1)",
"dest_name": "dkfut19",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT20",
"source_type": "CHAR(1)",
"dest_name": "dkfut20",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKEMCD",
"source_type": "CHAR(5)",
"dest_name": "dkemcd",
"dest_type": "text",
"description": "Employee Code"
},
{
"source_name": "DKMINS",
"source_type": "CHAR(3)",
"dest_name": "dkmins",
"dest_type": "text",
"description": "Minor Sales Code"
},
{
"source_name": "DKTERR",
"source_type": "CHAR(4)",
"dest_name": "dkterr",
"dest_type": "text",
"description": "Territory Code"
},
{
"source_name": "DKORD#",
"source_type": "NUMERIC(9,0)",
"dest_name": "dkord#",
"dest_type": "numeric(9,0)",
"description": "Customer Order Number"
},
{
"source_name": "DKBCUS",
"source_type": "CHAR(8)",
"dest_name": "dkbcus",
"dest_type": "text",
"description": "Bill-to Cust#"
},
{
"source_name": "DKLABH",
"source_type": "NUMERIC(11,2)",
"dest_name": "dklabh",
"dest_type": "numeric(11,2)",
"description": "Labour Hours"
},
{
"source_name": "DKCPYT",
"source_type": "CHAR(1)",
"dest_name": "dkcpyt",
"dest_type": "text",
"description": "was copied to another GTRAN"
},
{
"source_name": "DKCPYF",
"source_type": "CHAR(1)",
"dest_name": "dkcpyf",
"dest_type": "text",
"description": "was copied from another GTRAN"
},
{
"source_name": "DKCLSC",
"source_type": "CHAR(1)",
"dest_name": "dkclsc",
"dest_type": "text",
"description": "Class Code"
},
{
"source_name": "DKCFAN",
"source_type": "NUMERIC(12,0)",
"dest_name": "dkcfan",
"dest_type": "numeric(12,0)",
"description": "Copied-from Company & Account #"
},
{
"source_name": "DKJRNN",
"source_type": "NUMERIC(5,0)",
"dest_name": "dkjrnn",
"dest_type": "numeric(5,0)",
"description": "Journal Number"
},
{
"source_name": "DKMJSC",
"source_type": "CHAR(3)",
"dest_name": "dkmjsc",
"dest_type": "text",
"description": "Major Sales Code"
},
{
"source_name": "DKBCID",
"source_type": "NUMERIC(12,0)",
"dest_name": "dkbcid",
"dest_type": "numeric(12,0)",
"description": "Tracking ID"
},
{
"source_name": "DKBSRF",
"source_type": "CHAR(1)",
"dest_name": "dkbsrf",
"dest_type": "text",
"description": "Reconciliation Status"
},
{
"source_name": "DKRCID",
"source_type": "NUMERIC(12,0)",
"dest_name": "dkrcid",
"dest_type": "numeric(12,0)",
"description": "Record ID"
},
{
"source_name": "DKTMS",
"source_type": "TIMESTMP",
"dest_name": "dktms",
"dest_type": "text",
"description": "Date/Time Stamp"
},
{
"source_name": "DKVEND",
"source_type": "CHAR(10)",
"dest_name": "dkvend",
"dest_type": "text",
"description": "Vendor Code"
},
{
"source_name": "DKADRN",
"source_type": "NUMERIC(8,0)",
"dest_name": "dkadrn",
"dest_type": "numeric(8,0)",
"description": "OneTime Vendor#"
},
{
"source_name": "DKUSR",
"source_type": "CHAR(10)",
"dest_name": "dkusr",
"dest_type": "text",
"description": "User I.D."
},
{
"source_name": "DKWTYET",
"source_type": "NUMERIC(9,0)",
"dest_name": "dkwtyet",
"dest_type": "numeric(9,0)",
"description": "AR Month-End WTYET Key"
},
{
"source_name": "DKXRTE2",
"source_type": "NUMERIC(11,6)",
"dest_name": "dkxrte2",
"dest_type": "numeric(11,6)",
"description": "Exchange Rate 2"
},
{
"source_name": "DKXRTE3",
"source_type": "NUMERIC(11,6)",
"dest_name": "dkxrte3",
"dest_type": "numeric(11,6)",
"description": "Exchange Rate 3"
},
{
"source_name": "DKJECU",
"source_type": "CHAR(2)",
"dest_name": "dkjecu",
"dest_type": "text",
"description": "Entry Currency"
},
{
"source_name": "DKILIN",
"source_type": "NUMERIC(3,0)",
"dest_name": "dkilin",
"dest_type": "numeric(3,0)",
"description": "Invoice Line"
},
{
"source_name": "DKJREFC",
"source_type": "CHAR(20)",
"dest_name": "dkjrefc",
"dest_type": "text",
"description": "Journal Ref Code"
},
{
"source_name": "DKDKENT",
"source_type": "CHAR(30)",
"dest_name": "dkdkent",
"dest_type": "text",
"description": "Doc Link Entry Numbers"
},
{
"source_name": "DKARBTC",
"source_type": "NUMERIC(2,0)",
"dest_name": "dkarbtc",
"dest_type": "numeric(2,0)",
"description": "A/R Details Batch"
},
{
"source_name": "DKDTLP",
"source_type": "CHAR(1)",
"dest_name": "dkdtlp",
"dest_type": "text",
"description": "Detail Posting"
},
{
"source_name": "DKCFBR",
"source_type": "CHAR(1)",
"dest_name": "dkcfbr",
"dest_type": "text",
"description": "Created for Bank Stat Reconciliation"
},
{
"source_name": "DKPBTID",
"source_type": "NUMERIC(9,0)",
"dest_name": "dkpbtid",
"dest_type": "numeric(9,0)",
"description": "RPRH Batch#"
},
{
"source_name": "DKPBTID2",
"source_type": "NUMERIC(9,0)",
"dest_name": "dkpbtid2",
"dest_type": "numeric(9,0)",
"description": "PD829WH Batch#"
},
{
"source_name": "DKLBTID",
"source_type": "NUMERIC(9,0)",
"dest_name": "dklbtid",
"dest_type": "numeric(9,0)",
"description": "PD439WH Batch#"
},
{
"source_name": "DKLBTID2",
"source_type": "NUMERIC(9,0)",
"dest_name": "dklbtid2",
"dest_type": "numeric(9,0)",
"description": "LBRH Batch#"
},
{
"source_name": "DKFUT21",
"source_type": "CHAR(1)",
"dest_name": "dkfut21",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT22",
"source_type": "CHAR(1)",
"dest_name": "dkfut22",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT23",
"source_type": "CHAR(10)",
"dest_name": "dkfut23",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT24",
"source_type": "CHAR(10)",
"dest_name": "dkfut24",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT25",
"source_type": "CHAR(10)",
"dest_name": "dkfut25",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT26",
"source_type": "CHAR(20)",
"dest_name": "dkfut26",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT27",
"source_type": "CHAR(20)",
"dest_name": "dkfut27",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT28",
"source_type": "CHAR(30)",
"dest_name": "dkfut28",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT29",
"source_type": "CHAR(30)",
"dest_name": "dkfut29",
"dest_type": "text",
"description": "Future Use"
},
{
"source_name": "DKFUT30",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkfut30",
"dest_type": "numeric(15,5)",
"description": "Future Use"
},
{
"source_name": "DKFUT31",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkfut31",
"dest_type": "numeric(15,5)",
"description": "Future Use"
},
{
"source_name": "DKFUT32",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkfut32",
"dest_type": "numeric(15,5)",
"description": "Future Use"
},
{
"source_name": "DKFUT33",
"source_type": "NUMERIC(15,5)",
"dest_name": "dkfut33",
"dest_type": "numeric(15,5)",
"description": "Future Use"
}
],
"watermarks": [
{
"name": "gtran_tms",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dktms), '1900-01-01 00:00:00.000000') FROM cms.gtran",
"default_value": "1900-01-01 00:00:00.000000"
}
],
"hooks": [
{
"run_order": 0,
"connection": "usmidsap02",
"sql": "-- GTRAN <-> GLMT reconciliation (reference only; hook output is not captured by pipekit).\n-- Per CLOSED account-period: SUM(cms.gtran.dkamt) should equal cms.glmt AJ4TT{period}.\n-- account = aj4comp*10^10 + aj4gl#1*10^6 + aj4gl#2 ; year = dkfsyr*100+dkfsyy ; period = dkfspr.\n-- Current (max) period excluded (in flux). Known residual: a few 2026 period-10 accounts (9352*) differ by small amounts.\nWITH gt AS (\n SELECT \"dkacc#\" acct, (dkfsyr*100+dkfsyy)::int yr, dkfspr::int period, SUM(dkamt) s\n FROM cms.gtran WHERE (dkfsyr*100+dkfsyy) BETWEEN 2000 AND 2030\n GROUP BY \"dkacc#\", (dkfsyr*100+dkfsyy)::int, dkfspr::int\n), gl AS (\n SELECT aj4comp*10000000000 + \"aj4gl#1\"*1000000 + \"aj4gl#2\" acct, aj4ccyy::int yr, v.period, v.net\n FROM cms.glmt CROSS JOIN LATERAL (VALUES\n (1,aj4tt01),(2,aj4tt02),(3,aj4tt03),(4,aj4tt04),(5,aj4tt05),(6,aj4tt06),(7,aj4tt07),(8,aj4tt08),\n (9,aj4tt09),(10,aj4tt10),(11,aj4tt11),(12,aj4tt12),(13,aj4tt13),(14,aj4tt14),(15,aj4tt15)) v(period,net)\n), recon AS (\n SELECT gt.acct, gt.yr, gt.period, gt.s gtran_sum, COALESCE(gl.net,0) glmt_net\n FROM gt LEFT JOIN gl ON gl.acct=gt.acct AND gl.yr=gt.yr AND gl.period=gt.period\n WHERE gt.yr*100+gt.period < (SELECT max(yr*100+period) FROM gt)\n AND abs(gt.s - COALESCE(gl.net,0)) > 0.005\n)\nSELECT\n (SELECT count(*) FROM gt WHERE yr*100+period < (SELECT max(yr*100+period) FROM gt)) AS closed_periods_checked,\n (SELECT count(*) FROM recon) AS mismatches,\n (SELECT COALESCE(sum(abs(gtran_sum-glmt_net)),0) FROM recon) AS total_abs_diff,\n (SELECT string_agg(acct::text||chr(47)||yr||chr(47)||period||' d='||round(gtran_sum-glmt_net,2)::text, '; ')\n FROM (SELECT * FROM recon ORDER BY abs(gtran_sum-glmt_net) DESC LIMIT 5) z) AS worst",
"run_on": "success"
}
]
}
Reference in New Issue View Git Blame Copy Permalink