pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
feat/dest-preflight
pipekit/config/modules/sop30300.json
Paul Trowbridge dd9f89f4b5 feat: version-control pipeline definitions via export/apply 
Add `pipekit export` / `pipekit apply` (pipekit/config_io.py) to serialise the
DB's config — drivers, connections, modules (+ columns, watermarks, hooks),
groups, schedules — to a git-trackable text tree under config/, and rehydrate
it. SQLite stays runtime state; definitions become diffable/reviewable/
revertible.

- config only: run_log/group_run/settings and per-run state columns excluded
- name-keyed refs (portable across databases); source_query in .sql sidecars;
  columns as real JSON arrays for line-by-line diffs
- newline-normalised so CRLF-vs-LF is never a spurious change
- apply is create/update by name; child collections fully synced; top-level
  deletes gated behind --prune; --dry-run prints the plan
- round-trip is identity (export -> apply --dry-run == nothing to do)

Commits the current config/ as the first baseline, capturing the freshly
populated columns_json for rm00101/rm00301/iv00101. Documented in SPEC.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 10:56:55 -04:00

870 lines
21 KiB
JSON
Raw Permalink Blame History

{
"name": "sop30300",
"source_connection": "sqlserver",
"dest_connection": "usmidsap02",
"dest_table": "gp.sop30300",
"staging_table": "pipekit_staging.sop30300",
"merge_strategy": "incremental",
"merge_key": "soptype, sopnumbe, lnitmseq, cmpntseq",
"enabled": 1,
"dest_description": null,
"columns": [
{
"source_name": "SOPTYPE",
"source_type": "SMALLINT",
"dest_name": "soptype",
"dest_type": "smallint",
"description": null
},
{
"source_name": "SOPNUMBE",
"source_type": "CHAR(21)",
"dest_name": "sopnumbe",
"dest_type": "text",
"description": null
},
{
"source_name": "LNITMSEQ",
"source_type": "INT",
"dest_name": "lnitmseq",
"dest_type": "integer",
"description": null
},
{
"source_name": "CMPNTSEQ",
"source_type": "INT",
"dest_name": "cmpntseq",
"dest_type": "integer",
"description": null
},
{
"source_name": "ITEMNMBR",
"source_type": "CHAR(31)",
"dest_name": "itemnmbr",
"dest_type": "text",
"description": null
},
{
"source_name": "ITEMDESC",
"source_type": "CHAR(101)",
"dest_name": "itemdesc",
"dest_type": "text",
"description": null
},
{
"source_name": "NONINVEN",
"source_type": "SMALLINT",
"dest_name": "noninven",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DROPSHIP",
"source_type": "SMALLINT",
"dest_name": "dropship",
"dest_type": "smallint",
"description": null
},
{
"source_name": "UOFM",
"source_type": "CHAR(9)",
"dest_name": "uofm",
"dest_type": "text",
"description": null
},
{
"source_name": "LOCNCODE",
"source_type": "CHAR(11)",
"dest_name": "locncode",
"dest_type": "text",
"description": null
},
{
"source_name": "UNITCOST",
"source_type": "NUMERIC(19,5)",
"dest_name": "unitcost",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORUNTCST",
"source_type": "NUMERIC(19,5)",
"dest_name": "oruntcst",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "UNITPRCE",
"source_type": "NUMERIC(19,5)",
"dest_name": "unitprce",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORUNTPRC",
"source_type": "NUMERIC(19,5)",
"dest_name": "oruntprc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "XTNDPRCE",
"source_type": "NUMERIC(19,5)",
"dest_name": "xtndprce",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OXTNDPRC",
"source_type": "NUMERIC(19,5)",
"dest_name": "oxtndprc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "REMPRICE",
"source_type": "NUMERIC(19,5)",
"dest_name": "remprice",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OREPRICE",
"source_type": "NUMERIC(19,5)",
"dest_name": "oreprice",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "EXTDCOST",
"source_type": "NUMERIC(19,5)",
"dest_name": "extdcost",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OREXTCST",
"source_type": "NUMERIC(19,5)",
"dest_name": "orextcst",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "MRKDNAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "mrkdnamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORMRKDAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "ormrkdam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "MRKDNPCT",
"source_type": "SMALLINT",
"dest_name": "mrkdnpct",
"dest_type": "smallint",
"description": null
},
{
"source_name": "MRKDNTYP",
"source_type": "SMALLINT",
"dest_name": "mrkdntyp",
"dest_type": "smallint",
"description": null
},
{
"source_name": "INVINDX",
"source_type": "INT",
"dest_name": "invindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "CSLSINDX",
"source_type": "INT",
"dest_name": "cslsindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "SLSINDX",
"source_type": "INT",
"dest_name": "slsindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "MKDNINDX",
"source_type": "INT",
"dest_name": "mkdnindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "RTNSINDX",
"source_type": "INT",
"dest_name": "rtnsindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "INUSINDX",
"source_type": "INT",
"dest_name": "inusindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "INSRINDX",
"source_type": "INT",
"dest_name": "insrindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "DMGDINDX",
"source_type": "INT",
"dest_name": "dmgdindx",
"dest_type": "integer",
"description": null
},
{
"source_name": "ITMTSHID",
"source_type": "CHAR(15)",
"dest_name": "itmtshid",
"dest_type": "text",
"description": null
},
{
"source_name": "IVITMTXB",
"source_type": "SMALLINT",
"dest_name": "ivitmtxb",
"dest_type": "smallint",
"description": null
},
{
"source_name": "BKTSLSAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "bktslsam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORBKTSLS",
"source_type": "NUMERIC(19,5)",
"dest_name": "orbktsls",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TAXAMNT",
"source_type": "NUMERIC(19,5)",
"dest_name": "taxamnt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORTAXAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "ortaxamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "TXBTXAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "txbtxamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "OTAXTAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "otaxtamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "BSIVCTTL",
"source_type": "TINYINT",
"dest_name": "bsivcttl",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TRDISAMT",
"source_type": "NUMERIC(19,5)",
"dest_name": "trdisamt",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORTDISAM",
"source_type": "NUMERIC(19,5)",
"dest_name": "ortdisam",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "DISCSALE",
"source_type": "NUMERIC(19,5)",
"dest_name": "discsale",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ORDAVSLS",
"source_type": "NUMERIC(19,5)",
"dest_name": "ordavsls",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QUANTITY",
"source_type": "NUMERIC(19,5)",
"dest_name": "quantity",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ATYALLOC",
"source_type": "NUMERIC(19,5)",
"dest_name": "atyalloc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYINSVC",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyinsvc",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYINUSE",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyinuse",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYDMGED",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtydmged",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYRTRND",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyrtrnd",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYONHND",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyonhnd",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYCANCE",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtycance",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYCANOT",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtycanot",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYORDER",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyorder",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYPRBAC",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyprbac",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYPRBOO",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyprboo",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYPRINV",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyprinv",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYPRORD",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyprord",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYPRVRECVD",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyprvrecvd",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYRECVD",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyrecvd",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYREMAI",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyremai",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYREMBO",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyrembo",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYTBAOR",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtytbaor",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYTOINV",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtytoinv",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYTORDR",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtytordr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYFULFI",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyfulfi",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYSLCTD",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtyslctd",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "QTYBSUOM",
"source_type": "NUMERIC(19,5)",
"dest_name": "qtybsuom",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "EXTQTYAL",
"source_type": "NUMERIC(19,5)",
"dest_name": "extqtyal",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "EXTQTYSEL",
"source_type": "NUMERIC(19,5)",
"dest_name": "extqtysel",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "ReqShipDate",
"source_type": "DATETIME",
"dest_name": "reqshipdate",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "FUFILDAT",
"source_type": "DATETIME",
"dest_name": "fufildat",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "ACTLSHIP",
"source_type": "DATETIME",
"dest_name": "actlship",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "SHIPMTHD",
"source_type": "CHAR(15)",
"dest_name": "shipmthd",
"dest_type": "text",
"description": null
},
{
"source_name": "SALSTERR",
"source_type": "CHAR(15)",
"dest_name": "salsterr",
"dest_type": "text",
"description": null
},
{
"source_name": "SLPRSNID",
"source_type": "CHAR(15)",
"dest_name": "slprsnid",
"dest_type": "text",
"description": null
},
{
"source_name": "PRCLEVEL",
"source_type": "CHAR(11)",
"dest_name": "prclevel",
"dest_type": "text",
"description": null
},
{
"source_name": "COMMNTID",
"source_type": "CHAR(15)",
"dest_name": "commntid",
"dest_type": "text",
"description": null
},
{
"source_name": "BRKFLD1",
"source_type": "SMALLINT",
"dest_name": "brkfld1",
"dest_type": "smallint",
"description": null
},
{
"source_name": "BRKFLD2",
"source_type": "SMALLINT",
"dest_name": "brkfld2",
"dest_type": "smallint",
"description": null
},
{
"source_name": "BRKFLD3",
"source_type": "SMALLINT",
"dest_name": "brkfld3",
"dest_type": "smallint",
"description": null
},
{
"source_name": "CURRNIDX",
"source_type": "SMALLINT",
"dest_name": "currnidx",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TRXSORCE",
"source_type": "CHAR(13)",
"dest_name": "trxsorce",
"dest_type": "text",
"description": null
},
{
"source_name": "SOPLNERR",
"source_type": "BINARY",
"dest_name": "soplnerr",
"dest_type": "bytea",
"description": null
},
{
"source_name": "DOCNCORR",
"source_type": "CHAR(21)",
"dest_name": "docncorr",
"dest_type": "text",
"description": null
},
{
"source_name": "ORGSEQNM",
"source_type": "INT",
"dest_name": "orgseqnm",
"dest_type": "integer",
"description": null
},
{
"source_name": "ITEMCODE",
"source_type": "CHAR(15)",
"dest_name": "itemcode",
"dest_type": "text",
"description": null
},
{
"source_name": "PURCHSTAT",
"source_type": "SMALLINT",
"dest_name": "purchstat",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DECPLQTY",
"source_type": "SMALLINT",
"dest_name": "decplqty",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DECPLCUR",
"source_type": "SMALLINT",
"dest_name": "decplcur",
"dest_type": "smallint",
"description": null
},
{
"source_name": "ODECPLCU",
"source_type": "SMALLINT",
"dest_name": "odecplcu",
"dest_type": "smallint",
"description": null
},
{
"source_name": "EXCEPTIONALDEMAND",
"source_type": "TINYINT",
"dest_name": "exceptionaldemand",
"dest_type": "smallint",
"description": null
},
{
"source_name": "TAXSCHID",
"source_type": "CHAR(15)",
"dest_name": "taxschid",
"dest_type": "text",
"description": null
},
{
"source_name": "TXSCHSRC",
"source_type": "SMALLINT",
"dest_name": "txschsrc",
"dest_type": "smallint",
"description": null
},
{
"source_name": "PRSTADCD",
"source_type": "CHAR(15)",
"dest_name": "prstadcd",
"dest_type": "text",
"description": null
},
{
"source_name": "ShipToName",
"source_type": "CHAR(65)",
"dest_name": "shiptoname",
"dest_type": "text",
"description": null
},
{
"source_name": "CNTCPRSN",
"source_type": "CHAR(61)",
"dest_name": "cntcprsn",
"dest_type": "text",
"description": null
},
{
"source_name": "ADDRESS1",
"source_type": "CHAR(61)",
"dest_name": "address1",
"dest_type": "text",
"description": null
},
{
"source_name": "ADDRESS2",
"source_type": "CHAR(61)",
"dest_name": "address2",
"dest_type": "text",
"description": null
},
{
"source_name": "ADDRESS3",
"source_type": "CHAR(61)",
"dest_name": "address3",
"dest_type": "text",
"description": null
},
{
"source_name": "CITY",
"source_type": "CHAR(35)",
"dest_name": "city",
"dest_type": "text",
"description": null
},
{
"source_name": "STATE",
"source_type": "CHAR(29)",
"dest_name": "state",
"dest_type": "text",
"description": null
},
{
"source_name": "ZIPCODE",
"source_type": "CHAR(11)",
"dest_name": "zipcode",
"dest_type": "text",
"description": null
},
{
"source_name": "CCode",
"source_type": "CHAR(7)",
"dest_name": "ccode",
"dest_type": "text",
"description": null
},
{
"source_name": "COUNTRY",
"source_type": "CHAR(61)",
"dest_name": "country",
"dest_type": "text",
"description": null
},
{
"source_name": "PHONE1",
"source_type": "CHAR(21)",
"dest_name": "phone1",
"dest_type": "text",
"description": null
},
{
"source_name": "PHONE2",
"source_type": "CHAR(21)",
"dest_name": "phone2",
"dest_type": "text",
"description": null
},
{
"source_name": "PHONE3",
"source_type": "CHAR(21)",
"dest_name": "phone3",
"dest_type": "text",
"description": null
},
{
"source_name": "FAXNUMBR",
"source_type": "CHAR(21)",
"dest_name": "faxnumbr",
"dest_type": "text",
"description": null
},
{
"source_name": "Flags",
"source_type": "SMALLINT",
"dest_name": "flags",
"dest_type": "smallint",
"description": null
},
{
"source_name": "CONTNBR",
"source_type": "CHAR(11)",
"dest_name": "contnbr",
"dest_type": "text",
"description": null
},
{
"source_name": "CONTLNSEQNBR",
"source_type": "NUMERIC(19,5)",
"dest_name": "contlnseqnbr",
"dest_type": "numeric(19,5)",
"description": null
},
{
"source_name": "CONTSTARTDTE",
"source_type": "DATETIME",
"dest_name": "contstartdte",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "CONTENDDTE",
"source_type": "DATETIME",
"dest_name": "contenddte",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "CONTITEMNBR",
"source_type": "CHAR(31)",
"dest_name": "contitemnbr",
"dest_type": "text",
"description": null
},
{
"source_name": "CONTSERIALNBR",
"source_type": "CHAR(21)",
"dest_name": "contserialnbr",
"dest_type": "text",
"description": null
},
{
"source_name": "ISLINEINTRA",
"source_type": "TINYINT",
"dest_name": "islineintra",
"dest_type": "smallint",
"description": null
},
{
"source_name": "Print_Phone_NumberGB",
"source_type": "SMALLINT",
"dest_name": "print_phone_numbergb",
"dest_type": "smallint",
"description": null
},
{
"source_name": "DEX_ROW_TS",
"source_type": "DATETIME",
"dest_name": "dex_row_ts",
"dest_type": "timestamp",
"description": null
},
{
"source_name": "DEX_ROW_ID",
"source_type": "INT",
"dest_name": "dex_row_id",
"dest_type": "integer",
"description": null
}
],
"watermarks": [
{
"name": "sop30300_ts",
"connection": "usmidsap02",
"resolver_sql": "SELECT COALESCE(MAX(dex_row_ts) - INTERVAL '1 day', '1900-01-01') FROM gp.sop30300",
"default_value": null
}
],
"hooks": []
}
Reference in New Issue View Git Blame Copy Permalink