From 13de00bdf26c3de48c0b25604a42ae2d32bd33c5 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Mon, 25 Jun 2018 00:01:05 -0400 Subject: [PATCH] move everything into one folder --- LICENSE | 21 -- build_maps.xlsm => database/build_maps.xlsm | Bin .../deploy}/reload/dcard/curl_dcard.cmd | 0 .../deploy}/reload/dcard/d.csv | 0 .../deploy}/reload/dcard/extract.sql | 0 .../deploy}/reload/dcard/map.json | 0 .../deploy}/reload/dcard/srce.json | 0 .../deploy}/reload/dcard/vals.json | 0 .../deploy}/reload/hunt/curl_hunt.cmd | 0 .../deploy}/reload/hunt/extract.sql | 0 .../deploy}/reload/hunt/map.json | 0 .../deploy}/reload/hunt/srce.json | 0 .../deploy}/reload/paycom/extract.sql | 0 .../deploy}/reload/paycom/load.cmd | 0 .../deploy}/reload/paycom/map.json | 0 .../deploy}/reload/paycom/srce.json | 0 .../deploy}/reload/paycom/vals.json | 0 .../deploy}/reload/pncc/extract.sql | 0 .../deploy}/reload/pncc/load.cmd | 0 .../deploy}/reload/pncc/map.json | 0 .../deploy}/reload/pncc/srce.json | 0 .../deploy}/reload/pncc/vals.json | 0 .../deploy}/reload/pncl/extract.sql | 0 .../deploy}/reload/pncl/load.cmd | 0 .../deploy}/reload/pncl/srce.json | 0 .../deploy}/reload/pnco/extract.sql | 0 .../deploy}/reload/pnco/load.cmd | 0 .../deploy}/reload/pnco/srce.json | 0 .../deploy}/reload/wmpd/curl_hunt.cmd | 0 .../deploy}/reload/wmpd/extract.sql | 0 .../deploy}/reload/wmpd/srce.json | 0 {deploy => database/deploy}/setup.sql | 0 .../interface}/import/map_trigger.sql | 0 .../interface}/import/srce_import.sql | 0 .../interface}/import/srce_import_do.sql | 0 .../interface}/map_def/srce_map_def_set.sql | 0 .../map_def/srce_map_def_set_single.sql | 0 .../interface}/map_def/test_regex.sql | 0 .../interface}/map_def/test_regex_recs.sql | 0 .../interface}/map_values/map_rv_set.sql | 0 .../interface}/map_values/report_unmapped.sql | 0 .../map_values/report_unmapped_recs.sql | 0 .../map_values/srce_map_overwrite.sql | 0 .../source_maint/srce_build_view.sql | 0 .../interface}/source_maint/srce_set.sql | 0 readme.md => database/readme.md | 254 +++++++++--------- .../reports}/all_map_return_values.sql | 0 .../reports}/colateral_balalance.sql | 0 {reports => database/reports}/dcard_bal.sql | 0 {reports => database/reports}/key_list.sql | 0 {reports => database/reports}/list_maps.sql | 0 .../reports}/loan_balance.sql | 0 {templates => database/templates}/regex.json | 0 {templates => database/templates}/srce.json | 0 package.json | 19 -- 55 files changed, 127 insertions(+), 167 deletions(-) delete mode 100644 LICENSE rename build_maps.xlsm => database/build_maps.xlsm (100%) rename {deploy => database/deploy}/reload/dcard/curl_dcard.cmd (100%) rename {deploy => database/deploy}/reload/dcard/d.csv (100%) rename {deploy => database/deploy}/reload/dcard/extract.sql (100%) rename {deploy => database/deploy}/reload/dcard/map.json (100%) rename {deploy => database/deploy}/reload/dcard/srce.json (100%) rename {deploy => database/deploy}/reload/dcard/vals.json (100%) rename {deploy => database/deploy}/reload/hunt/curl_hunt.cmd (100%) rename {deploy => database/deploy}/reload/hunt/extract.sql (100%) rename {deploy => database/deploy}/reload/hunt/map.json (100%) rename {deploy => database/deploy}/reload/hunt/srce.json (100%) rename {deploy => database/deploy}/reload/paycom/extract.sql (100%) rename {deploy => database/deploy}/reload/paycom/load.cmd (100%) rename {deploy => database/deploy}/reload/paycom/map.json (100%) rename {deploy => database/deploy}/reload/paycom/srce.json (100%) rename {deploy => database/deploy}/reload/paycom/vals.json (100%) rename {deploy => database/deploy}/reload/pncc/extract.sql (100%) rename {deploy => database/deploy}/reload/pncc/load.cmd (100%) rename {deploy => database/deploy}/reload/pncc/map.json (100%) rename {deploy => database/deploy}/reload/pncc/srce.json (100%) rename {deploy => database/deploy}/reload/pncc/vals.json (100%) rename {deploy => database/deploy}/reload/pncl/extract.sql (100%) rename {deploy => database/deploy}/reload/pncl/load.cmd (100%) rename {deploy => database/deploy}/reload/pncl/srce.json (100%) rename {deploy => database/deploy}/reload/pnco/extract.sql (100%) rename {deploy => database/deploy}/reload/pnco/load.cmd (100%) rename {deploy => database/deploy}/reload/pnco/srce.json (100%) rename {deploy => database/deploy}/reload/wmpd/curl_hunt.cmd (100%) rename {deploy => database/deploy}/reload/wmpd/extract.sql (100%) rename {deploy => database/deploy}/reload/wmpd/srce.json (100%) rename {deploy => database/deploy}/setup.sql (100%) rename {interface => database/interface}/import/map_trigger.sql (100%) rename {interface => database/interface}/import/srce_import.sql (100%) rename {interface => database/interface}/import/srce_import_do.sql (100%) rename {interface => database/interface}/map_def/srce_map_def_set.sql (100%) rename {interface => database/interface}/map_def/srce_map_def_set_single.sql (100%) rename {interface => database/interface}/map_def/test_regex.sql (100%) rename {interface => database/interface}/map_def/test_regex_recs.sql (100%) rename {interface => database/interface}/map_values/map_rv_set.sql (100%) rename {interface => database/interface}/map_values/report_unmapped.sql (100%) rename {interface => database/interface}/map_values/report_unmapped_recs.sql (100%) rename {interface => database/interface}/map_values/srce_map_overwrite.sql (100%) rename {interface => database/interface}/source_maint/srce_build_view.sql (100%) rename {interface => database/interface}/source_maint/srce_set.sql (100%) rename readme.md => database/readme.md (97%) rename {reports => database/reports}/all_map_return_values.sql (100%) rename {reports => database/reports}/colateral_balalance.sql (100%) rename {reports => database/reports}/dcard_bal.sql (100%) rename {reports => database/reports}/key_list.sql (100%) rename {reports => database/reports}/list_maps.sql (100%) rename {reports => database/reports}/loan_balance.sql (100%) rename {templates => database/templates}/regex.json (100%) rename {templates => database/templates}/srce.json (100%) delete mode 100644 package.json diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 911ea4a..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/build_maps.xlsm b/database/build_maps.xlsm similarity index 100% rename from build_maps.xlsm rename to database/build_maps.xlsm diff --git a/deploy/reload/dcard/curl_dcard.cmd b/database/deploy/reload/dcard/curl_dcard.cmd similarity index 100% rename from deploy/reload/dcard/curl_dcard.cmd rename to database/deploy/reload/dcard/curl_dcard.cmd diff --git a/deploy/reload/dcard/d.csv b/database/deploy/reload/dcard/d.csv similarity index 100% rename from deploy/reload/dcard/d.csv rename to database/deploy/reload/dcard/d.csv diff --git a/deploy/reload/dcard/extract.sql b/database/deploy/reload/dcard/extract.sql similarity index 100% rename from deploy/reload/dcard/extract.sql rename to database/deploy/reload/dcard/extract.sql diff --git a/deploy/reload/dcard/map.json b/database/deploy/reload/dcard/map.json similarity index 100% rename from deploy/reload/dcard/map.json rename to database/deploy/reload/dcard/map.json diff --git a/deploy/reload/dcard/srce.json b/database/deploy/reload/dcard/srce.json similarity index 100% rename from deploy/reload/dcard/srce.json rename to database/deploy/reload/dcard/srce.json diff --git a/deploy/reload/dcard/vals.json b/database/deploy/reload/dcard/vals.json similarity index 100% rename from deploy/reload/dcard/vals.json rename to database/deploy/reload/dcard/vals.json diff --git a/deploy/reload/hunt/curl_hunt.cmd b/database/deploy/reload/hunt/curl_hunt.cmd similarity index 100% rename from deploy/reload/hunt/curl_hunt.cmd rename to database/deploy/reload/hunt/curl_hunt.cmd diff --git a/deploy/reload/hunt/extract.sql b/database/deploy/reload/hunt/extract.sql similarity index 100% rename from deploy/reload/hunt/extract.sql rename to database/deploy/reload/hunt/extract.sql diff --git a/deploy/reload/hunt/map.json b/database/deploy/reload/hunt/map.json similarity index 100% rename from deploy/reload/hunt/map.json rename to database/deploy/reload/hunt/map.json diff --git a/deploy/reload/hunt/srce.json b/database/deploy/reload/hunt/srce.json similarity index 100% rename from deploy/reload/hunt/srce.json rename to database/deploy/reload/hunt/srce.json diff --git a/deploy/reload/paycom/extract.sql b/database/deploy/reload/paycom/extract.sql similarity index 100% rename from deploy/reload/paycom/extract.sql rename to database/deploy/reload/paycom/extract.sql diff --git a/deploy/reload/paycom/load.cmd b/database/deploy/reload/paycom/load.cmd similarity index 100% rename from deploy/reload/paycom/load.cmd rename to database/deploy/reload/paycom/load.cmd diff --git a/deploy/reload/paycom/map.json b/database/deploy/reload/paycom/map.json similarity index 100% rename from deploy/reload/paycom/map.json rename to database/deploy/reload/paycom/map.json diff --git a/deploy/reload/paycom/srce.json b/database/deploy/reload/paycom/srce.json similarity index 100% rename from deploy/reload/paycom/srce.json rename to database/deploy/reload/paycom/srce.json diff --git a/deploy/reload/paycom/vals.json b/database/deploy/reload/paycom/vals.json similarity index 100% rename from deploy/reload/paycom/vals.json rename to database/deploy/reload/paycom/vals.json diff --git a/deploy/reload/pncc/extract.sql b/database/deploy/reload/pncc/extract.sql similarity index 100% rename from deploy/reload/pncc/extract.sql rename to database/deploy/reload/pncc/extract.sql diff --git a/deploy/reload/pncc/load.cmd b/database/deploy/reload/pncc/load.cmd similarity index 100% rename from deploy/reload/pncc/load.cmd rename to database/deploy/reload/pncc/load.cmd diff --git a/deploy/reload/pncc/map.json b/database/deploy/reload/pncc/map.json similarity index 100% rename from deploy/reload/pncc/map.json rename to database/deploy/reload/pncc/map.json diff --git a/deploy/reload/pncc/srce.json b/database/deploy/reload/pncc/srce.json similarity index 100% rename from deploy/reload/pncc/srce.json rename to database/deploy/reload/pncc/srce.json diff --git a/deploy/reload/pncc/vals.json b/database/deploy/reload/pncc/vals.json similarity index 100% rename from deploy/reload/pncc/vals.json rename to database/deploy/reload/pncc/vals.json diff --git a/deploy/reload/pncl/extract.sql b/database/deploy/reload/pncl/extract.sql similarity index 100% rename from deploy/reload/pncl/extract.sql rename to database/deploy/reload/pncl/extract.sql diff --git a/deploy/reload/pncl/load.cmd b/database/deploy/reload/pncl/load.cmd similarity index 100% rename from deploy/reload/pncl/load.cmd rename to database/deploy/reload/pncl/load.cmd diff --git a/deploy/reload/pncl/srce.json b/database/deploy/reload/pncl/srce.json similarity index 100% rename from deploy/reload/pncl/srce.json rename to database/deploy/reload/pncl/srce.json diff --git a/deploy/reload/pnco/extract.sql b/database/deploy/reload/pnco/extract.sql similarity index 100% rename from deploy/reload/pnco/extract.sql rename to database/deploy/reload/pnco/extract.sql diff --git a/deploy/reload/pnco/load.cmd b/database/deploy/reload/pnco/load.cmd similarity index 100% rename from deploy/reload/pnco/load.cmd rename to database/deploy/reload/pnco/load.cmd diff --git a/deploy/reload/pnco/srce.json b/database/deploy/reload/pnco/srce.json similarity index 100% rename from deploy/reload/pnco/srce.json rename to database/deploy/reload/pnco/srce.json diff --git a/deploy/reload/wmpd/curl_hunt.cmd b/database/deploy/reload/wmpd/curl_hunt.cmd similarity index 100% rename from deploy/reload/wmpd/curl_hunt.cmd rename to database/deploy/reload/wmpd/curl_hunt.cmd diff --git a/deploy/reload/wmpd/extract.sql b/database/deploy/reload/wmpd/extract.sql similarity index 100% rename from deploy/reload/wmpd/extract.sql rename to database/deploy/reload/wmpd/extract.sql diff --git a/deploy/reload/wmpd/srce.json b/database/deploy/reload/wmpd/srce.json similarity index 100% rename from deploy/reload/wmpd/srce.json rename to database/deploy/reload/wmpd/srce.json diff --git a/deploy/setup.sql b/database/deploy/setup.sql similarity index 100% rename from deploy/setup.sql rename to database/deploy/setup.sql diff --git a/interface/import/map_trigger.sql b/database/interface/import/map_trigger.sql similarity index 100% rename from interface/import/map_trigger.sql rename to database/interface/import/map_trigger.sql diff --git a/interface/import/srce_import.sql b/database/interface/import/srce_import.sql similarity index 100% rename from interface/import/srce_import.sql rename to database/interface/import/srce_import.sql diff --git a/interface/import/srce_import_do.sql b/database/interface/import/srce_import_do.sql similarity index 100% rename from interface/import/srce_import_do.sql rename to database/interface/import/srce_import_do.sql diff --git a/interface/map_def/srce_map_def_set.sql b/database/interface/map_def/srce_map_def_set.sql similarity index 100% rename from interface/map_def/srce_map_def_set.sql rename to database/interface/map_def/srce_map_def_set.sql diff --git a/interface/map_def/srce_map_def_set_single.sql b/database/interface/map_def/srce_map_def_set_single.sql similarity index 100% rename from interface/map_def/srce_map_def_set_single.sql rename to database/interface/map_def/srce_map_def_set_single.sql diff --git a/interface/map_def/test_regex.sql b/database/interface/map_def/test_regex.sql similarity index 100% rename from interface/map_def/test_regex.sql rename to database/interface/map_def/test_regex.sql diff --git a/interface/map_def/test_regex_recs.sql b/database/interface/map_def/test_regex_recs.sql similarity index 100% rename from interface/map_def/test_regex_recs.sql rename to database/interface/map_def/test_regex_recs.sql diff --git a/interface/map_values/map_rv_set.sql b/database/interface/map_values/map_rv_set.sql similarity index 100% rename from interface/map_values/map_rv_set.sql rename to database/interface/map_values/map_rv_set.sql diff --git a/interface/map_values/report_unmapped.sql b/database/interface/map_values/report_unmapped.sql similarity index 100% rename from interface/map_values/report_unmapped.sql rename to database/interface/map_values/report_unmapped.sql diff --git a/interface/map_values/report_unmapped_recs.sql b/database/interface/map_values/report_unmapped_recs.sql similarity index 100% rename from interface/map_values/report_unmapped_recs.sql rename to database/interface/map_values/report_unmapped_recs.sql diff --git a/interface/map_values/srce_map_overwrite.sql b/database/interface/map_values/srce_map_overwrite.sql similarity index 100% rename from interface/map_values/srce_map_overwrite.sql rename to database/interface/map_values/srce_map_overwrite.sql diff --git a/interface/source_maint/srce_build_view.sql b/database/interface/source_maint/srce_build_view.sql similarity index 100% rename from interface/source_maint/srce_build_view.sql rename to database/interface/source_maint/srce_build_view.sql diff --git a/interface/source_maint/srce_set.sql b/database/interface/source_maint/srce_set.sql similarity index 100% rename from interface/source_maint/srce_set.sql rename to database/interface/source_maint/srce_set.sql diff --git a/readme.md b/database/readme.md similarity index 97% rename from readme.md rename to database/readme.md index 71e3a1f..4cb3cc9 100644 --- a/readme.md +++ b/database/readme.md @@ -1,128 +1,128 @@ -Generic Data Transformation Tool -======================================================= - -The goal is to: -1. house external data and prevent duplication on insert -2. facilitate regular exression operations to extract meaningful data -3. be able to reference it from outside sources (no action required) and maintain reference to original data - - -It is well suited for data from outside systems that -* requires complex transformation (parsing and mapping) -* original data is retained for reference -* don't feel like writing a map-reduce - -use cases: -* on-going bank feeds -* jumbled product lists -* storing api results - - -The data is converted to json by the importing program and inserted to the database. -Regex expressions are applied to specified json components and the results can be mapped to other values. - - -Major Interactions ------------------------- - -* Source Definitions (Maint/Inquire) -* Regex Instructions (Maint/Inquire) -* Cross Reference List (Maint/Inquire) -* Run Import (Run Job) - - - -### Interaction Details -* _Source Definitions (Maint/Inquire)_ - - * display a list of existing sources with display detials/edit options - * create new option - * underlying function is `tps.srce_set(_name text, _defn jsonb)` - - * the current definition of a source includes data based on bad presumptions: - * how to load from a csv file using `COPY` - * setup a Postgres type to reflect the associated columns (if applicable) - - -* _Regex Instructions (Maint/Inquire)_ - - * display a list of existing instruction sets with display details/edit options - * create new option - * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json - -* _Cross Reference List (Maint/Inquire)_ - - * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` - * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs - * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` - -* _Run Import_ - - * underlying function is `tps.srce_import(_path text, _srce text)` - - - -source definition ----------------------------------------------------------------------- - -* **load data** - * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` - * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function - * build constraint key `based on srce definition` - * handle violations - * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) - * build an import log - * run maps (as opposed to relying on trigger) -* **read data** - * the `schema` key contains either a text element or a text array in curly braces - * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` - * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table - * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` - * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` - * view that `uses the source definiton` to extrapolate a table? - * a materialized table is built `based on the source definition` and any addtional regex? - * add regex = alter table add column with historic updates? - * no primary key? - * every document must work out to one row - -``` -{ - "name":"dcard", - "source":"client_file", - "loading_function":"csv" - "constraint":[ - "{Trans. Date}", - "{Post Date}" - ], - "schemas":{ - "default":[ - { - "path":"{doc,origin_addresses,0}", - "type":"text", - "column_name":"origin_address" - }, - { - "path":"{doc,destination_addresses,0}", - "type":"text", - "column_name":"origin_address" - }, - { - "path":"{doc,status}", - "type":"text", - "column_name":"status" - } - { - "path":"{doc,rows,0,elements,0,distance,value}", - "type":"numeric", - "column_name":"distance" - } - { - "path":"{doc,rows,0,elements,0,duration,value}", - "type":"numeric", - "column_name":"duration" - } - ], - "version2":[] - } -} +Generic Data Transformation Tool +======================================================= + +The goal is to: +1. house external data and prevent duplication on insert +2. facilitate regular exression operations to extract meaningful data +3. be able to reference it from outside sources (no action required) and maintain reference to original data + + +It is well suited for data from outside systems that +* requires complex transformation (parsing and mapping) +* original data is retained for reference +* don't feel like writing a map-reduce + +use cases: +* on-going bank feeds +* jumbled product lists +* storing api results + + +The data is converted to json by the importing program and inserted to the database. +Regex expressions are applied to specified json components and the results can be mapped to other values. + + +Major Interactions +------------------------ + +* Source Definitions (Maint/Inquire) +* Regex Instructions (Maint/Inquire) +* Cross Reference List (Maint/Inquire) +* Run Import (Run Job) + + + +### Interaction Details +* _Source Definitions (Maint/Inquire)_ + + * display a list of existing sources with display detials/edit options + * create new option + * underlying function is `tps.srce_set(_name text, _defn jsonb)` + + * the current definition of a source includes data based on bad presumptions: + * how to load from a csv file using `COPY` + * setup a Postgres type to reflect the associated columns (if applicable) + + +* _Regex Instructions (Maint/Inquire)_ + + * display a list of existing instruction sets with display details/edit options + * create new option + * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json + +* _Cross Reference List (Maint/Inquire)_ + + * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` + * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs + * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` + +* _Run Import_ + + * underlying function is `tps.srce_import(_path text, _srce text)` + + + +source definition +---------------------------------------------------------------------- + +* **load data** + * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` + * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function + * build constraint key `based on srce definition` + * handle violations + * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) + * build an import log + * run maps (as opposed to relying on trigger) +* **read data** + * the `schema` key contains either a text element or a text array in curly braces + * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` + * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table + * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` + * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` + * view that `uses the source definiton` to extrapolate a table? + * a materialized table is built `based on the source definition` and any addtional regex? + * add regex = alter table add column with historic updates? + * no primary key? + * every document must work out to one row + +``` +{ + "name":"dcard", + "source":"client_file", + "loading_function":"csv" + "constraint":[ + "{Trans. Date}", + "{Post Date}" + ], + "schemas":{ + "default":[ + { + "path":"{doc,origin_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,destination_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,status}", + "type":"text", + "column_name":"status" + } + { + "path":"{doc,rows,0,elements,0,distance,value}", + "type":"numeric", + "column_name":"distance" + } + { + "path":"{doc,rows,0,elements,0,duration,value}", + "type":"numeric", + "column_name":"duration" + } + ], + "version2":[] + } +} ``` \ No newline at end of file diff --git a/reports/all_map_return_values.sql b/database/reports/all_map_return_values.sql similarity index 100% rename from reports/all_map_return_values.sql rename to database/reports/all_map_return_values.sql diff --git a/reports/colateral_balalance.sql b/database/reports/colateral_balalance.sql similarity index 100% rename from reports/colateral_balalance.sql rename to database/reports/colateral_balalance.sql diff --git a/reports/dcard_bal.sql b/database/reports/dcard_bal.sql similarity index 100% rename from reports/dcard_bal.sql rename to database/reports/dcard_bal.sql diff --git a/reports/key_list.sql b/database/reports/key_list.sql similarity index 100% rename from reports/key_list.sql rename to database/reports/key_list.sql diff --git a/reports/list_maps.sql b/database/reports/list_maps.sql similarity index 100% rename from reports/list_maps.sql rename to database/reports/list_maps.sql diff --git a/reports/loan_balance.sql b/database/reports/loan_balance.sql similarity index 100% rename from reports/loan_balance.sql rename to database/reports/loan_balance.sql diff --git a/templates/regex.json b/database/templates/regex.json similarity index 100% rename from templates/regex.json rename to database/templates/regex.json diff --git a/templates/srce.json b/database/templates/srce.json similarity index 100% rename from templates/srce.json rename to database/templates/srce.json diff --git a/package.json b/package.json deleted file mode 100644 index 38f783d..0000000 --- a/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "tps_etl", - "version": "1.0.0", - "description": "third party source data transformation", - "main": "index.js", - "scripts": { - "test": "uh" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/fleetside72/tps_etl.git" - }, - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/fleetside72/tps_etl/issues" - }, - "homepage": "https://github.com/fleetside72/tps_etl#readme" -}