From 3484e8ea7b705306015dac46c0fe2a673cd4bbcb Mon Sep 17 00:00:00 2001 From: Antonio Rivero <38889534+Antonio-RiveroMartnez@users.noreply.github.com> Date: Fri, 24 Feb 2023 14:36:21 -0300 Subject: [PATCH] feat(ssh_tunnel): Import/Export Databases with SSHTunnel credentials (#23099) --- docs/static/resources/openapi.json | 72 +++ .../ImportModal/ImportModal.test.tsx | 47 ++ .../src/components/ImportModal/index.tsx | 188 +++++++- .../src/pages/ChartList/index.tsx | 18 + .../views/CRUD/dashboard/DashboardList.tsx | 18 + .../data/database/DatabaseModal/index.tsx | 190 +++++++- .../views/CRUD/data/dataset/DatasetList.tsx | 18 + .../CRUD/data/savedquery/SavedQueryList.tsx | 18 + superset-frontend/src/views/CRUD/hooks.ts | 47 ++ .../src/views/CRUD/utils.test.tsx | 163 +++++++ superset-frontend/src/views/CRUD/utils.tsx | 66 ++- superset/charts/api.py | 46 +- superset/commands/importers/v1/__init__.py | 17 +- superset/commands/importers/v1/assets.py | 17 +- superset/commands/importers/v1/utils.py | 60 +++ superset/dashboards/api.py | 47 +- superset/databases/api.py | 46 +- superset/databases/commands/export.py | 10 + .../databases/commands/importers/v1/utils.py | 8 + superset/databases/schemas.py | 72 ++- .../ssh_tunnel/commands/exceptions.py | 8 + superset/databases/ssh_tunnel/models.py | 13 + superset/datasets/api.py | 42 ++ superset/datasets/commands/export.py | 11 + superset/importexport/api.py | 47 +- superset/queries/saved_queries/api.py | 46 +- .../integration_tests/databases/api_tests.py | 448 ++++++++++++++++++ .../databases/commands_tests.py | 191 ++++++++ .../fixtures/importexport.py | 107 +++++ tests/unit_tests/importexport/api_test.py | 8 +- 30 files changed, 2039 insertions(+), 50 deletions(-) diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index f303b83784..7729d413da 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -10716,6 +10716,18 @@ "passwords": { "description": "JSON map of passwords for each featured database in the ZIP file. If the ZIP includes a database config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", "type": "string" + }, + "ssh_tunnel_passwords": { + "description": "JSON map of passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keys": { + "description": "JSON map of private_keys for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keyspasswords": { + "description": "JSON map of private_key_passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key_password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key_password\"}`.", + "type": "string" } }, "type": "object" @@ -11439,6 +11451,18 @@ "passwords": { "description": "JSON map of passwords for each featured database in the ZIP file. If the ZIP includes a database config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", "type": "string" + }, + "ssh_tunnel_passwords": { + "description": "JSON map of passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keys": { + "description": "JSON map of private_keys for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keyspasswords": { + "description": "JSON map of private_key_passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key_password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key_password\"}`.", + "type": "string" } }, "type": "object" @@ -13020,6 +13044,18 @@ "passwords": { "description": "JSON map of passwords for each featured database in the ZIP file. If the ZIP includes a database config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", "type": "string" + }, + "ssh_tunnel_passwords": { + "description": "JSON map of passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keys": { + "description": "JSON map of private_keys for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keyspasswords": { + "description": "JSON map of private_key_passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key_password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key_password\"}`.", + "type": "string" } }, "type": "object" @@ -14788,6 +14824,18 @@ "passwords": { "description": "JSON map of passwords for each featured database in the ZIP file. If the ZIP includes a database config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", "type": "string" + }, + "ssh_tunnel_passwords": { + "description": "JSON map of passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keys": { + "description": "JSON map of private_keys for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keyspasswords": { + "description": "JSON map of private_key_passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key_password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key_password\"}`.", + "type": "string" } }, "type": "object" @@ -16231,6 +16279,18 @@ "sync_metrics": { "description": "sync metrics?", "type": "boolean" + }, + "ssh_tunnel_passwords": { + "description": "JSON map of passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keys": { + "description": "JSON map of private_keys for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keyspasswords": { + "description": "JSON map of private_key_passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key_password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key_password\"}`.", + "type": "string" } }, "type": "object" @@ -19428,6 +19488,18 @@ "passwords": { "description": "JSON map of passwords for each featured database in the ZIP file. If the ZIP includes a database config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", "type": "string" + }, + "ssh_tunnel_passwords": { + "description": "JSON map of passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_password\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keys": { + "description": "JSON map of private_keys for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key\"}`.", + "type": "string" + }, + "ssh_tunnel_private_keyspasswords": { + "description": "JSON map of private_key_passwords for each ssh_tunnel associated to a featured database in the ZIP file. If the ZIP includes a ssh_tunnel config in the path `databases/MyDatabase.yaml`, the private_key_password should be provided in the following format: `{\"databases/MyDatabase.yaml\": \"my_private_key_password\"}`.", + "type": "string" } }, "type": "object" diff --git a/superset-frontend/src/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/components/ImportModal/ImportModal.test.tsx index f86f0bff8e..862c918d05 100644 --- a/superset-frontend/src/components/ImportModal/ImportModal.test.tsx +++ b/superset-frontend/src/components/ImportModal/ImportModal.test.tsx @@ -146,4 +146,51 @@ describe('ImportModelsModal', () => { ); expect(wrapperWithPasswords.find('input[type="password"]')).toExist(); }); + + it('should render ssh_tunnel password fields when needed for import', () => { + const wrapperWithPasswords = mount( + , + { + context: { store }, + }, + ); + expect( + wrapperWithPasswords.find('[data-test="ssh_tunnel_password"]'), + ).toExist(); + }); + + it('should render ssh_tunnel private_key fields when needed for import', () => { + const wrapperWithPasswords = mount( + , + { + context: { store }, + }, + ); + expect( + wrapperWithPasswords.find('[data-test="ssh_tunnel_private_key"]'), + ).toExist(); + }); + + it('should render ssh_tunnel private_key_password fields when needed for import', () => { + const wrapperWithPasswords = mount( + , + { + context: { store }, + }, + ); + expect( + wrapperWithPasswords.find( + '[data-test="ssh_tunnel_private_key_password"]', + ), + ).toExist(); + }); }); diff --git a/superset-frontend/src/components/ImportModal/index.tsx b/superset-frontend/src/components/ImportModal/index.tsx index 4e8ec11396..274bff948f 100644 --- a/superset-frontend/src/components/ImportModal/index.tsx +++ b/superset-frontend/src/components/ImportModal/index.tsx @@ -110,6 +110,14 @@ export interface ImportModelsModalProps { onHide: () => void; passwordFields?: string[]; setPasswordFields?: (passwordFields: string[]) => void; + sshTunnelPasswordFields?: string[]; + setSSHTunnelPasswordFields?: (sshTunnelPasswordFields: string[]) => void; + sshTunnelPrivateKeyFields?: string[]; + setSSHTunnelPrivateKeyFields?: (sshTunnelPrivateKeyFields: string[]) => void; + sshTunnelPrivateKeyPasswordFields?: string[]; + setSSHTunnelPrivateKeyPasswordFields?: ( + sshTunnelPrivateKeyPasswordFields: string[], + ) => void; } const ImportModelsModal: FunctionComponent = ({ @@ -122,6 +130,12 @@ const ImportModelsModal: FunctionComponent = ({ onHide, passwordFields = [], setPasswordFields = () => {}, + sshTunnelPasswordFields = [], + setSSHTunnelPasswordFields = () => {}, + sshTunnelPrivateKeyFields = [], + setSSHTunnelPrivateKeyFields = () => {}, + sshTunnelPrivateKeyPasswordFields = [], + setSSHTunnelPrivateKeyPasswordFields = () => {}, }) => { const [isHidden, setIsHidden] = useState(true); const [passwords, setPasswords] = useState>({}); @@ -131,6 +145,14 @@ const ImportModelsModal: FunctionComponent = ({ const [fileList, setFileList] = useState([]); const [importingModel, setImportingModel] = useState(false); const [errorMessage, setErrorMessage] = useState(); + const [sshTunnelPasswords, setSSHTunnelPasswords] = useState< + Record + >({}); + const [sshTunnelPrivateKeys, setSSHTunnelPrivateKeys] = useState< + Record + >({}); + const [sshTunnelPrivateKeyPasswords, setSSHTunnelPrivateKeyPasswords] = + useState>({}); const clearModal = () => { setFileList([]); @@ -140,6 +162,12 @@ const ImportModelsModal: FunctionComponent = ({ setConfirmedOverwrite(false); setImportingModel(false); setErrorMessage(''); + setSSHTunnelPasswordFields([]); + setSSHTunnelPrivateKeyFields([]); + setSSHTunnelPrivateKeyPasswordFields([]); + setSSHTunnelPasswords({}); + setSSHTunnelPrivateKeys({}); + setSSHTunnelPrivateKeyPasswords({}); }; const handleErrorMsg = (msg: string) => { @@ -147,7 +175,13 @@ const ImportModelsModal: FunctionComponent = ({ }; const { - state: { alreadyExists, passwordsNeeded }, + state: { + alreadyExists, + passwordsNeeded, + sshPasswordNeeded, + sshPrivateKeyNeeded, + sshPrivateKeyPasswordNeeded, + }, importResource, } = useImportResource(resourceName, resourceLabel, handleErrorMsg); @@ -165,6 +199,27 @@ const ImportModelsModal: FunctionComponent = ({ } }, [alreadyExists, setNeedsOverwriteConfirm]); + useEffect(() => { + setSSHTunnelPasswordFields(sshPasswordNeeded); + if (sshPasswordNeeded.length > 0) { + setImportingModel(false); + } + }, [sshPasswordNeeded, setSSHTunnelPasswordFields]); + + useEffect(() => { + setSSHTunnelPrivateKeyFields(sshPrivateKeyNeeded); + if (sshPrivateKeyNeeded.length > 0) { + setImportingModel(false); + } + }, [sshPrivateKeyNeeded, setSSHTunnelPrivateKeyFields]); + + useEffect(() => { + setSSHTunnelPrivateKeyPasswordFields(sshPrivateKeyPasswordNeeded); + if (sshPrivateKeyPasswordNeeded.length > 0) { + setImportingModel(false); + } + }, [sshPrivateKeyPasswordNeeded, setSSHTunnelPrivateKeyPasswordFields]); + // Functions const hide = () => { setIsHidden(true); @@ -181,6 +236,9 @@ const ImportModelsModal: FunctionComponent = ({ importResource( fileList[0].originFileObj, passwords, + sshTunnelPasswords, + sshTunnelPrivateKeys, + sshTunnelPrivateKeyPasswords, confirmedOverwrite, ).then(result => { if (result) { @@ -210,30 +268,117 @@ const ImportModelsModal: FunctionComponent = ({ }; const renderPasswordFields = () => { - if (passwordFields.length === 0) { + if ( + passwordFields.length === 0 && + sshTunnelPasswordFields.length === 0 && + sshTunnelPrivateKeyFields.length === 0 && + sshTunnelPrivateKeyPasswordFields.length === 0 + ) { return null; } + const files = [ + ...new Set([ + ...passwordFields, + ...sshTunnelPasswordFields, + ...sshTunnelPrivateKeyFields, + ...sshTunnelPrivateKeyPasswordFields, + ]), + ]; + return ( <>
{t('Database passwords')}
{passwordsNeededMessage} - {passwordFields.map(fileName => ( - -
- {fileName} - * -
- - setPasswords({ ...passwords, [fileName]: event.target.value }) - } - /> -
+ {files.map(fileName => ( + <> + {passwordFields?.indexOf(fileName) >= 0 && ( + +
+ {t('%s PASSWORD', fileName.slice(10))} + * +
+ + setPasswords({ + ...passwords, + [fileName]: event.target.value, + }) + } + /> +
+ )} + {sshTunnelPasswordFields?.indexOf(fileName) >= 0 && ( + +
+ {t('%s SSH TUNNEL PASSWORD', fileName.slice(10))} + * +
+ + setSSHTunnelPasswords({ + ...sshTunnelPasswords, + [fileName]: event.target.value, + }) + } + data-test="ssh_tunnel_password" + /> +
+ )} + {sshTunnelPrivateKeyFields?.indexOf(fileName) >= 0 && ( + +
+ {t('%s SSH TUNNEL PRIVATE KEY', fileName.slice(10))} + * +
+