46 lines
1009 B
Markdown
46 lines
1009 B
Markdown
|
|
||
|
|
Data movement:
|
||
|
|
- Kafka
|
||
|
|
- FDW
|
||
|
|
- remote server (MSSQL)
|
||
|
|
- jrunner
|
||
|
|
- dbt
|
||
|
|
|
||
|
|
|
||
|
|
J Runner
|
||
|
|
----------
|
||
|
|
|
||
|
|
takes a yaml
|
||
|
|
- connections #parameter
|
||
|
|
- script to build the SQL script #file path or SQL
|
||
|
|
- the sql script used for extraction #file path or SQL
|
||
|
|
- destination table sized for the extracted data #parameter
|
||
|
|
- ingestion script to merge incoming data with final destination data #file path or SQL
|
||
|
|
|
||
|
|
```yaml
|
||
|
|
database:
|
||
|
|
source:
|
||
|
|
uri: jdbc:mysql://localhost:3306/source_db
|
||
|
|
username: source_user
|
||
|
|
password: source_password
|
||
|
|
target:
|
||
|
|
uri: jdbc:postgresql://localhost:5432/target_db
|
||
|
|
username: target_user
|
||
|
|
password: target_password
|
||
|
|
|
||
|
|
script:
|
||
|
|
path: /path/to/sql_script.sql
|
||
|
|
|
||
|
|
logging:
|
||
|
|
level: INFO
|
||
|
|
file: /path/to/logfile.log
|
||
|
|
|
||
|
|
runtime:
|
||
|
|
java_options:
|
||
|
|
- -Xms512m
|
||
|
|
- -Xmx2048m
|
||
|
|
environment_variables:
|
||
|
|
JAVA_HOME: /path/to/custom/java
|
||
|
|
|
||
|
|
```
|