mirror of https://github.com/apache/superset.git
feat(dbc ui): Adding Google Sheets Dynamic Form (#15801)
* feat: Make Google Sheets Dyanmic (#15576) * first draft * second draft * added tests * first draft * added table_catalog * remove console.log * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * revisions * save this for now * working form * save disable on public sheets * refactor somethings * saving this for now * working edit * add back query to schema * working add * fix styling * fixing x * fix linting * prettier * fix some type issues * more lint fixes * remove unused dependency * fix linint * fix validation * pylint bypass * pylint bypass * fix this * fix mypy * yerp * fix test * fix test * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * wrap add sheets * fix linting issues * fix unit test * ignore typing * fix editting and paste issues * remove query * fix this * fix test * add test back * fix error messaging * update url messaging on error * change error type * Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx Co-authored-by: Beto Dealmeida <roberto@dealmeida.net> * add errors for sheets with no name * fix * fix messaging for gsheets * stop pylint * update line Co-authored-by: AAfghahi <48933336+AAfghahi@users.noreply.github.com> Co-authored-by: Arash <arash.afghahi@gmail.com> Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>
This commit is contained in:
parent
3f6c81b621
commit
bfe7eb9a7b
|
@ -217,7 +217,7 @@ max-nested-blocks=5
|
|||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=88
|
||||
max-line-length=90
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
|
|
@ -19,19 +19,21 @@
|
|||
import React, { FormEvent, useState } from 'react';
|
||||
import { SupersetTheme, JsonObject, t } from '@superset-ui/core';
|
||||
import { InputProps } from 'antd/lib/input';
|
||||
import { Switch, Select, Button } from 'src/common/components';
|
||||
import { Input, Switch, Select, Button } from 'src/common/components';
|
||||
import InfoTooltip from 'src/components/InfoTooltip';
|
||||
import ValidatedInput from 'src/components/Form/LabeledErrorBoundInput';
|
||||
import FormLabel from 'src/components/Form/FormLabel';
|
||||
import { DeleteFilled } from '@ant-design/icons';
|
||||
import { DeleteFilled, CloseOutlined } from '@ant-design/icons';
|
||||
import {
|
||||
formScrollableStyles,
|
||||
validatedFormStyles,
|
||||
CredentialInfoForm,
|
||||
toggleStyle,
|
||||
infoTooltip,
|
||||
StyledFooterButton,
|
||||
StyledCatalogTable,
|
||||
} from './styles';
|
||||
import { DatabaseForm, DatabaseObject } from '../types';
|
||||
import { CatalogObject, DatabaseForm, DatabaseObject } from '../types';
|
||||
|
||||
enum CredentialInfoOptions {
|
||||
jsonUpload,
|
||||
|
@ -46,6 +48,7 @@ export const FormFieldOrder = [
|
|||
'password',
|
||||
'database_name',
|
||||
'credentials_info',
|
||||
'catalog',
|
||||
'query',
|
||||
'encryption',
|
||||
];
|
||||
|
@ -58,7 +61,10 @@ interface FieldPropTypes {
|
|||
onParametersUploadFileChange: (value: any) => string;
|
||||
changeMethods: { onParametersChange: (value: any) => string } & {
|
||||
onChange: (value: any) => string;
|
||||
} & { onParametersUploadFileChange: (value: any) => string };
|
||||
} & { onParametersUploadFileChange: (value: any) => string } & {
|
||||
onAddTableCatalog: () => void;
|
||||
onRemoveTableCatalog: (idx: number) => void;
|
||||
};
|
||||
validationErrors: JsonObject | null;
|
||||
getValidation: () => void;
|
||||
db?: DatabaseObject;
|
||||
|
@ -187,6 +193,89 @@ const CredentialsInfo = ({
|
|||
);
|
||||
};
|
||||
|
||||
const TableCatalog = ({
|
||||
required,
|
||||
changeMethods,
|
||||
getValidation,
|
||||
validationErrors,
|
||||
db,
|
||||
}: FieldPropTypes) => {
|
||||
const tableCatalog = db?.catalog || [];
|
||||
const catalogError = validationErrors || {};
|
||||
return (
|
||||
<StyledCatalogTable>
|
||||
<div className="catalog-type-select">
|
||||
<FormLabel required>{t('Type of Google Sheets Allowed')}</FormLabel>
|
||||
<Select style={{ width: '100%' }} defaultValue="true" disabled>
|
||||
<Select.Option value="true" key={1}>
|
||||
{t('Publicly shared sheets only')}
|
||||
</Select.Option>
|
||||
</Select>
|
||||
</div>
|
||||
<h4 className="gsheet-title">
|
||||
{t('Connect Google Sheets as tables to this database')}
|
||||
</h4>
|
||||
<div>
|
||||
{tableCatalog?.map((sheet: CatalogObject, idx: number) => (
|
||||
<>
|
||||
<FormLabel className="catalog-label" required>
|
||||
{t('Google Sheet Name and URL')}
|
||||
</FormLabel>
|
||||
<div className="catalog-name">
|
||||
<Input
|
||||
className="catalog-name-input"
|
||||
placeholder={t('Enter a name for this sheet')}
|
||||
onChange={e => {
|
||||
changeMethods.onParametersChange({
|
||||
target: {
|
||||
type: `catalog-${idx}`,
|
||||
name: 'name',
|
||||
value: e.target.value,
|
||||
},
|
||||
});
|
||||
}}
|
||||
value={sheet.name}
|
||||
/>
|
||||
|
||||
{tableCatalog?.length > 1 && (
|
||||
<CloseOutlined
|
||||
className="catalog-delete"
|
||||
onClick={() => changeMethods.onRemoveTableCatalog(idx)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<ValidatedInput
|
||||
className="catalog-name-url"
|
||||
required={required}
|
||||
validationMethods={{ onBlur: getValidation }}
|
||||
errorMessage={catalogError[sheet.name]}
|
||||
placeholder={t('Paste the shareable Google Sheet URL here')}
|
||||
onChange={(e: { target: { value: any } }) =>
|
||||
changeMethods.onParametersChange({
|
||||
target: {
|
||||
type: `catalog-${idx}`,
|
||||
name: 'value',
|
||||
value: e.target.value,
|
||||
},
|
||||
})
|
||||
}
|
||||
value={sheet.value}
|
||||
/>
|
||||
</>
|
||||
))}
|
||||
<StyledFooterButton
|
||||
className="catalog-add-btn"
|
||||
onClick={() => {
|
||||
changeMethods.onAddTableCatalog();
|
||||
}}
|
||||
>
|
||||
+ {t('Add sheet')}
|
||||
</StyledFooterButton>
|
||||
</div>
|
||||
</StyledCatalogTable>
|
||||
);
|
||||
};
|
||||
|
||||
const hostField = ({
|
||||
required,
|
||||
changeMethods,
|
||||
|
@ -300,18 +389,22 @@ const displayField = ({
|
|||
validationErrors,
|
||||
db,
|
||||
}: FieldPropTypes) => (
|
||||
<ValidatedInput
|
||||
id="database_name"
|
||||
name="database_name"
|
||||
required
|
||||
value={db?.database_name}
|
||||
validationMethods={{ onBlur: getValidation }}
|
||||
errorMessage={validationErrors?.database_name}
|
||||
placeholder=""
|
||||
label="Display Name"
|
||||
onChange={changeMethods.onChange}
|
||||
helpText={t('Pick a nickname for this database to display as in Superset.')}
|
||||
/>
|
||||
<>
|
||||
<ValidatedInput
|
||||
id="database_name"
|
||||
name="database_name"
|
||||
required
|
||||
value={db?.database_name}
|
||||
validationMethods={{ onBlur: getValidation }}
|
||||
errorMessage={validationErrors?.database_name}
|
||||
placeholder=""
|
||||
label={t('Display Name')}
|
||||
onChange={changeMethods.onChange}
|
||||
helpText={t(
|
||||
'Pick a nickname for this database to display as in Superset.',
|
||||
)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
|
||||
const queryField = ({
|
||||
|
@ -375,6 +468,7 @@ const FORM_FIELD_MAP = {
|
|||
query: queryField,
|
||||
encryption: forceSSLField,
|
||||
credentials_info: CredentialsInfo,
|
||||
catalog: TableCatalog,
|
||||
};
|
||||
|
||||
const DatabaseConnectionForm = ({
|
||||
|
@ -382,6 +476,8 @@ const DatabaseConnectionForm = ({
|
|||
onParametersChange,
|
||||
onChange,
|
||||
onParametersUploadFileChange,
|
||||
onAddTableCatalog,
|
||||
onRemoveTableCatalog,
|
||||
validationErrors,
|
||||
getValidation,
|
||||
db,
|
||||
|
@ -403,6 +499,8 @@ const DatabaseConnectionForm = ({
|
|||
onParametersUploadFileChange?: (
|
||||
event: FormEvent<InputProps> | { target: HTMLInputElement },
|
||||
) => void;
|
||||
onAddTableCatalog: () => void;
|
||||
onRemoveTableCatalog: (idx: number) => void;
|
||||
validationErrors: JsonObject | null;
|
||||
getValidation: () => void;
|
||||
}) => (
|
||||
|
@ -426,6 +524,8 @@ const DatabaseConnectionForm = ({
|
|||
onParametersChange,
|
||||
onChange,
|
||||
onParametersUploadFileChange,
|
||||
onAddTableCatalog,
|
||||
onRemoveTableCatalog,
|
||||
},
|
||||
validationErrors,
|
||||
getValidation,
|
||||
|
|
|
@ -50,6 +50,7 @@ import {
|
|||
DatabaseObject,
|
||||
DatabaseForm,
|
||||
CONFIGURATION_METHOD,
|
||||
CatalogObject,
|
||||
} from 'src/views/CRUD/data/database/types';
|
||||
import Loading from 'src/components/Loading';
|
||||
import ExtraOptions from './ExtraOptions';
|
||||
|
@ -101,10 +102,15 @@ const errorAlertMapping = {
|
|||
message: 'Invalid account information',
|
||||
description: 'Either the username or password is incorrect.',
|
||||
},
|
||||
INVALID_PAYLOAD_SCHEMA: {
|
||||
INVALID_PAYLOAD_SCHEMA_ERROR: {
|
||||
message: 'Incorrect Fields',
|
||||
description: 'Please make sure all fields are filled out correctly',
|
||||
},
|
||||
TABLE_DOES_NOT_EXIST_ERROR: {
|
||||
message: 'URL could not be identified',
|
||||
description:
|
||||
'The URL could not be identified. Please check for typos and make sure that "Type of google sheet allowed" selection matches the input',
|
||||
},
|
||||
};
|
||||
interface DatabaseModalProps {
|
||||
addDangerToast: (msg: string) => void;
|
||||
|
@ -126,6 +132,8 @@ enum ActionType {
|
|||
textChange,
|
||||
extraInputChange,
|
||||
extraEditorChange,
|
||||
addTableCatalogSheet,
|
||||
removeTableCatalogSheet,
|
||||
}
|
||||
|
||||
interface DBReducerPayloadType {
|
||||
|
@ -161,7 +169,13 @@ type DBReducerActionType =
|
|||
};
|
||||
}
|
||||
| {
|
||||
type: ActionType.reset;
|
||||
type: ActionType.reset | ActionType.addTableCatalogSheet;
|
||||
}
|
||||
| {
|
||||
type: ActionType.removeTableCatalogSheet;
|
||||
payload: {
|
||||
indexToDelete: number;
|
||||
};
|
||||
}
|
||||
| {
|
||||
type: ActionType.configMethodChange;
|
||||
|
@ -180,6 +194,9 @@ function dbReducer(
|
|||
...(state || {}),
|
||||
};
|
||||
let query = '';
|
||||
let deserializeExtraJSON = {};
|
||||
let extra_json: DatabaseObject['extra_json'];
|
||||
|
||||
switch (action.type) {
|
||||
case ActionType.extraEditorChange:
|
||||
return {
|
||||
|
@ -227,6 +244,29 @@ function dbReducer(
|
|||
[action.payload.name]: action.payload.value,
|
||||
};
|
||||
case ActionType.parametersChange:
|
||||
if (
|
||||
trimmedState.catalog !== undefined &&
|
||||
action.payload.type?.startsWith('catalog')
|
||||
) {
|
||||
// Formatting wrapping google sheets table catalog
|
||||
const idx = action.payload.type?.split('-')[1];
|
||||
const catalogToUpdate = trimmedState?.catalog[idx] || {};
|
||||
catalogToUpdate[action.payload.name] = action.payload.value;
|
||||
|
||||
const paramatersCatalog = {};
|
||||
// eslint-disable-next-line array-callback-return
|
||||
trimmedState.catalog?.map((item: CatalogObject) => {
|
||||
paramatersCatalog[item.name] = item.value;
|
||||
});
|
||||
|
||||
return {
|
||||
...trimmedState,
|
||||
parameters: {
|
||||
...trimmedState.parameters,
|
||||
catalog: paramatersCatalog,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
...trimmedState,
|
||||
parameters: {
|
||||
|
@ -234,6 +274,22 @@ function dbReducer(
|
|||
[action.payload.name]: action.payload.value,
|
||||
},
|
||||
};
|
||||
case ActionType.addTableCatalogSheet:
|
||||
if (trimmedState.catalog !== undefined) {
|
||||
return {
|
||||
...trimmedState,
|
||||
catalog: [...trimmedState.catalog, { name: '', value: '' }],
|
||||
};
|
||||
}
|
||||
return {
|
||||
...trimmedState,
|
||||
catalog: [{ name: '', value: '' }],
|
||||
};
|
||||
case ActionType.removeTableCatalogSheet:
|
||||
trimmedState.catalog?.splice(action.payload.indexToDelete, 1);
|
||||
return {
|
||||
...trimmedState,
|
||||
};
|
||||
case ActionType.editorChange:
|
||||
return {
|
||||
...trimmedState,
|
||||
|
@ -246,10 +302,8 @@ function dbReducer(
|
|||
};
|
||||
case ActionType.fetched:
|
||||
// convert all the keys in this payload into strings
|
||||
// eslint-disable-next-line no-case-declarations
|
||||
let deserializeExtraJSON = {};
|
||||
if (action.payload.extra) {
|
||||
const extra_json = {
|
||||
extra_json = {
|
||||
...JSON.parse(action.payload.extra || ''),
|
||||
} as DatabaseObject['extra_json'];
|
||||
|
||||
|
@ -262,13 +316,6 @@ function dbReducer(
|
|||
};
|
||||
}
|
||||
|
||||
if (action.payload?.parameters?.query) {
|
||||
// convert query into URI params string
|
||||
query = new URLSearchParams(
|
||||
action.payload.parameters.query as string,
|
||||
).toString();
|
||||
}
|
||||
|
||||
if (
|
||||
action.payload.backend === 'bigquery' &&
|
||||
action.payload.configuration_method ===
|
||||
|
@ -288,6 +335,46 @@ function dbReducer(
|
|||
};
|
||||
}
|
||||
|
||||
if (
|
||||
action.payload.backend === 'gsheets' &&
|
||||
action.payload.configuration_method ===
|
||||
CONFIGURATION_METHOD.DYNAMIC_FORM &&
|
||||
extra_json?.engine_params?.catalog !== undefined
|
||||
) {
|
||||
// pull catalog from engine params
|
||||
const engineParamsCatalog = extra_json?.engine_params?.catalog;
|
||||
|
||||
return {
|
||||
...action.payload,
|
||||
engine: action.payload.backend,
|
||||
configuration_method: action.payload.configuration_method,
|
||||
extra_json: deserializeExtraJSON,
|
||||
catalog: Object.keys(engineParamsCatalog).map(e => ({
|
||||
name: e,
|
||||
value: engineParamsCatalog[e],
|
||||
})),
|
||||
} as DatabaseObject;
|
||||
}
|
||||
|
||||
if (action.payload?.parameters?.query) {
|
||||
// convert query into URI params string
|
||||
query = new URLSearchParams(
|
||||
action.payload.parameters.query as string,
|
||||
).toString();
|
||||
|
||||
return {
|
||||
...action.payload,
|
||||
encrypted_extra: action.payload.encrypted_extra || '',
|
||||
engine: action.payload.backend || trimmedState.engine,
|
||||
configuration_method: action.payload.configuration_method,
|
||||
extra_json: deserializeExtraJSON,
|
||||
parameters: {
|
||||
...action.payload.parameters,
|
||||
query,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...action.payload,
|
||||
encrypted_extra: action.payload.encrypted_extra || '',
|
||||
|
@ -296,9 +383,9 @@ function dbReducer(
|
|||
extra_json: deserializeExtraJSON,
|
||||
parameters: {
|
||||
...action.payload.parameters,
|
||||
query,
|
||||
},
|
||||
};
|
||||
|
||||
case ActionType.dbSelected:
|
||||
return {
|
||||
...action.payload,
|
||||
|
@ -319,7 +406,9 @@ const serializeExtra = (extraJson: DatabaseObject['extra_json']) =>
|
|||
JSON.stringify({
|
||||
...extraJson,
|
||||
metadata_params: JSON.parse((extraJson?.metadata_params as string) || '{}'),
|
||||
engine_params: JSON.parse((extraJson?.engine_params as string) || '{}'),
|
||||
engine_params: JSON.parse(
|
||||
((extraJson?.engine_params as unknown) as string) || '{}',
|
||||
),
|
||||
schemas_allowed_for_csv_upload:
|
||||
(extraJson?.schemas_allowed_for_csv_upload as string) || '[]',
|
||||
});
|
||||
|
@ -369,7 +458,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
t('database'),
|
||||
addDangerToast,
|
||||
);
|
||||
|
||||
const isDynamic = (engine: string | undefined) =>
|
||||
availableDbs?.databases.filter(
|
||||
(DB: DatabaseObject) => DB.backend === engine || DB.engine === engine,
|
||||
|
@ -435,7 +523,10 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
.replace(/&/g, '","')
|
||||
.replace(/=/g, '":"')}"}`,
|
||||
);
|
||||
} else if (dbToUpdate?.parameters?.query === '') {
|
||||
} else if (
|
||||
dbToUpdate?.parameters?.query === '' &&
|
||||
'query' in dbModel.parameters
|
||||
) {
|
||||
dbToUpdate.parameters.query = {};
|
||||
}
|
||||
|
||||
|
@ -466,6 +557,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
}
|
||||
}
|
||||
|
||||
if (dbToUpdate.parameters.catalog) {
|
||||
// need to stringify gsheets catalog to allow it to be seralized
|
||||
dbToUpdate.extra_json = {
|
||||
engine_params: JSON.stringify({
|
||||
catalog: dbToUpdate.parameters.catalog,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
if (dbToUpdate?.extra_json) {
|
||||
// convert extra_json to back to string
|
||||
dbToUpdate.extra = serializeExtra(dbToUpdate?.extra_json);
|
||||
|
@ -545,6 +645,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
engine,
|
||||
},
|
||||
});
|
||||
setDB({ type: ActionType.addTableCatalogSheet });
|
||||
};
|
||||
|
||||
const renderAvailableSelector = () => (
|
||||
|
@ -816,6 +917,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
value: target.value,
|
||||
})
|
||||
}
|
||||
onAddTableCatalog={() =>
|
||||
setDB({ type: ActionType.addTableCatalogSheet })
|
||||
}
|
||||
onRemoveTableCatalog={(idx: number) =>
|
||||
setDB({
|
||||
type: ActionType.removeTableCatalogSheet,
|
||||
payload: { indexToDelete: idx },
|
||||
})
|
||||
}
|
||||
getValidation={() => getValidation(db)}
|
||||
validationErrors={validationErrors}
|
||||
/>
|
||||
|
@ -928,6 +1038,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
value: target.value,
|
||||
})
|
||||
}
|
||||
onAddTableCatalog={() =>
|
||||
setDB({ type: ActionType.addTableCatalogSheet })
|
||||
}
|
||||
onRemoveTableCatalog={(idx: number) =>
|
||||
setDB({
|
||||
type: ActionType.removeTableCatalogSheet,
|
||||
payload: { indexToDelete: idx },
|
||||
})
|
||||
}
|
||||
getValidation={() => getValidation(db)}
|
||||
validationErrors={validationErrors}
|
||||
/>
|
||||
|
@ -1030,7 +1149,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
</>
|
||||
) : (
|
||||
<>
|
||||
{/* Step 1 */}
|
||||
{/* Dyanmic Form Step 1 */}
|
||||
{!isLoading &&
|
||||
(!db ? (
|
||||
<SelectDatabaseStyles>
|
||||
|
@ -1073,6 +1192,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
|||
db={db}
|
||||
sslForced={sslForced}
|
||||
dbModel={dbModel}
|
||||
onAddTableCatalog={() => {
|
||||
setDB({ type: ActionType.addTableCatalogSheet });
|
||||
}}
|
||||
onRemoveTableCatalog={(idx: number) => {
|
||||
setDB({
|
||||
type: ActionType.removeTableCatalogSheet,
|
||||
payload: { indexToDelete: idx },
|
||||
});
|
||||
}}
|
||||
onParametersChange={({
|
||||
target,
|
||||
}: {
|
||||
|
|
|
@ -537,3 +537,43 @@ export const StyledStickyHeader = styled.div`
|
|||
z-index: ${({ theme }) => theme.zIndex.max};
|
||||
background: ${({ theme }) => theme.colors.grayscale.light5};
|
||||
`;
|
||||
|
||||
export const StyledCatalogTable = styled.div`
|
||||
margin-bottom: 16px;
|
||||
|
||||
.catalog-type-select {
|
||||
margin: 0 0 40px;
|
||||
}
|
||||
|
||||
.gsheet-title {
|
||||
font-size: ${({ theme }) => theme.typography.sizes.l * 1.1}px;
|
||||
font-weight: bold;
|
||||
margin: ${({ theme }) => theme.gridUnit * 6}px 0 16px;
|
||||
}
|
||||
|
||||
.catalog-label {
|
||||
margin: 0 0 8px;
|
||||
}
|
||||
|
||||
.catalog-name {
|
||||
display: flex;
|
||||
.catalog-name-input {
|
||||
width: 95%;
|
||||
}
|
||||
}
|
||||
|
||||
.catalog-name-url {
|
||||
margin: 4px 0;
|
||||
width: 95%;
|
||||
}
|
||||
|
||||
.catalog-delete {
|
||||
align-self: center;
|
||||
background: ${({ theme }) => theme.colors.grayscale.light4};
|
||||
margin: 5px;
|
||||
}
|
||||
|
||||
.catalog-add-btn {
|
||||
width: 95%;
|
||||
}
|
||||
`;
|
||||
|
|
|
@ -21,6 +21,11 @@ type DatabaseUser = {
|
|||
last_name: string;
|
||||
};
|
||||
|
||||
export type CatalogObject = {
|
||||
name: string;
|
||||
value: string;
|
||||
};
|
||||
|
||||
export type DatabaseObject = {
|
||||
// Connection + general
|
||||
id?: number;
|
||||
|
@ -41,10 +46,14 @@ export type DatabaseObject = {
|
|||
encryption?: boolean;
|
||||
credentials_info?: string;
|
||||
query?: string | object;
|
||||
catalog?: {};
|
||||
};
|
||||
configuration_method: CONFIGURATION_METHOD;
|
||||
engine?: string;
|
||||
|
||||
// Gsheets temporary storage
|
||||
catalog?: Array<CatalogObject>;
|
||||
|
||||
// Performance
|
||||
cache_timeout?: string;
|
||||
allow_run_async?: boolean;
|
||||
|
@ -65,7 +74,9 @@ export type DatabaseObject = {
|
|||
|
||||
// Extra
|
||||
extra_json?: {
|
||||
engine_params?: {} | string;
|
||||
engine_params?: {
|
||||
catalog: Record<any, any> | string;
|
||||
};
|
||||
metadata_params?: {} | string;
|
||||
metadata_cache_timeout?: {
|
||||
schema_cache_timeout?: number; // in Performance
|
||||
|
|
|
@ -674,7 +674,11 @@ export function useDatabaseValidation() {
|
|||
message,
|
||||
}: {
|
||||
error_type: string;
|
||||
extra: { invalid?: string[]; missing?: string[] };
|
||||
extra: {
|
||||
invalid?: string[];
|
||||
missing?: string[];
|
||||
name: string;
|
||||
};
|
||||
message: string;
|
||||
},
|
||||
) => {
|
||||
|
@ -682,6 +686,13 @@ export function useDatabaseValidation() {
|
|||
// error can't be mapped to a parameter
|
||||
// so leave it alone
|
||||
if (extra.invalid) {
|
||||
if (extra.invalid[0] === 'catalog') {
|
||||
return {
|
||||
...obj,
|
||||
[extra.name]: message,
|
||||
error_type,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...obj,
|
||||
[extra.invalid[0]]: message,
|
||||
|
|
|
@ -64,7 +64,7 @@ class ValidateDatabaseParametersCommand(BaseCommand):
|
|||
),
|
||||
)
|
||||
engine_spec = engine_specs[engine]
|
||||
if not issubclass(engine_spec, BasicParametersMixin):
|
||||
if not hasattr(engine_spec, "parameters_schema"):
|
||||
raise InvalidEngineError(
|
||||
SupersetError(
|
||||
message=__(
|
||||
|
@ -85,7 +85,9 @@ class ValidateDatabaseParametersCommand(BaseCommand):
|
|||
)
|
||||
|
||||
# perform initial validation
|
||||
errors = engine_spec.validate_parameters(self._properties.get("parameters", {}))
|
||||
errors = engine_spec.validate_parameters( # type: ignore
|
||||
self._properties.get("parameters", {})
|
||||
)
|
||||
if errors:
|
||||
raise InvalidParametersError(errors)
|
||||
|
||||
|
@ -96,9 +98,8 @@ class ValidateDatabaseParametersCommand(BaseCommand):
|
|||
encrypted_extra = {}
|
||||
|
||||
# try to connect
|
||||
sqlalchemy_uri = engine_spec.build_sqlalchemy_uri(
|
||||
self._properties.get("parameters"), # type: ignore
|
||||
encrypted_extra,
|
||||
sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( # type: ignore
|
||||
self._properties.get("parameters"), encrypted_extra,
|
||||
)
|
||||
if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri():
|
||||
sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted
|
||||
|
|
|
@ -19,13 +19,18 @@ import re
|
|||
from contextlib import closing
|
||||
from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
|
||||
|
||||
from apispec import APISpec
|
||||
from apispec.ext.marshmallow import MarshmallowPlugin
|
||||
from flask import g
|
||||
from flask_babel import gettext as __
|
||||
from marshmallow import fields, Schema
|
||||
from marshmallow.exceptions import ValidationError
|
||||
from sqlalchemy.engine import create_engine
|
||||
from sqlalchemy.engine.url import URL
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from superset import security_manager
|
||||
from superset.databases.schemas import encrypted_field_properties
|
||||
from superset.db_engine_specs.sqlite import SqliteEngineSpec
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
|
||||
|
@ -35,11 +40,16 @@ if TYPE_CHECKING:
|
|||
|
||||
SYNTAX_ERROR_REGEX = re.compile('SQLError: near "(?P<server_error>.*?)": syntax error')
|
||||
|
||||
ma_plugin = MarshmallowPlugin()
|
||||
|
||||
|
||||
class GSheetsParametersSchema(Schema):
|
||||
catalog = fields.Dict()
|
||||
|
||||
|
||||
class GSheetsParametersType(TypedDict):
|
||||
credentials_info: Dict[str, Any]
|
||||
query: Dict[str, Any]
|
||||
table_catalog: Dict[str, str]
|
||||
catalog: Dict[str, str]
|
||||
|
||||
|
||||
class GSheetsEngineSpec(SqliteEngineSpec):
|
||||
|
@ -50,6 +60,10 @@ class GSheetsEngineSpec(SqliteEngineSpec):
|
|||
allows_joins = True
|
||||
allows_subqueries = True
|
||||
|
||||
parameters_schema = GSheetsParametersSchema()
|
||||
default_driver = "apsw"
|
||||
sqlalchemy_uri_placeholder = "gsheets://"
|
||||
|
||||
custom_errors: Dict[Pattern[str], Tuple[str, SupersetErrorType, Dict[str, Any]]] = {
|
||||
SYNTAX_ERROR_REGEX: (
|
||||
__(
|
||||
|
@ -87,16 +101,64 @@ class GSheetsEngineSpec(SqliteEngineSpec):
|
|||
|
||||
return {"metadata": metadata["extra"]}
|
||||
|
||||
@classmethod
|
||||
def build_sqlalchemy_uri(
|
||||
cls,
|
||||
_: GSheetsParametersType,
|
||||
encrypted_extra: Optional[ # pylint: disable=unused-argument
|
||||
Dict[str, Any]
|
||||
] = None,
|
||||
) -> str: # pylint: disable=unused-variable
|
||||
|
||||
return "gsheets://"
|
||||
|
||||
@classmethod
|
||||
def get_parameters_from_uri(
|
||||
cls, encrypted_extra: Optional[Dict[str, str]] = None,
|
||||
) -> Any:
|
||||
# Building parameters from encrypted_extra and uri
|
||||
if encrypted_extra:
|
||||
return {**encrypted_extra}
|
||||
|
||||
raise ValidationError("Invalid service credentials")
|
||||
|
||||
@classmethod
|
||||
def parameters_json_schema(cls) -> Any:
|
||||
"""
|
||||
Return configuration parameters as OpenAPI.
|
||||
"""
|
||||
if not cls.parameters_schema:
|
||||
return None
|
||||
|
||||
spec = APISpec(
|
||||
title="Database Parameters",
|
||||
version="1.0.0",
|
||||
openapi_version="3.0.0",
|
||||
plugins=[ma_plugin],
|
||||
)
|
||||
|
||||
ma_plugin.init_spec(spec)
|
||||
ma_plugin.converter.add_attribute_function(encrypted_field_properties)
|
||||
spec.components.schema(cls.__name__, schema=cls.parameters_schema)
|
||||
return spec.to_dict()["components"]["schemas"][cls.__name__]
|
||||
|
||||
@classmethod
|
||||
def validate_parameters(
|
||||
cls, parameters: GSheetsParametersType,
|
||||
) -> List[SupersetError]:
|
||||
errors: List[SupersetError] = []
|
||||
|
||||
credentials_info = parameters.get("credentials_info")
|
||||
table_catalog = parameters.get("table_catalog", {})
|
||||
table_catalog = parameters.get("catalog", {})
|
||||
|
||||
if not table_catalog:
|
||||
errors.append(
|
||||
SupersetError(
|
||||
message="URL is required",
|
||||
error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={"invalid": ["catalog"], "name": "", "url": ""},
|
||||
),
|
||||
)
|
||||
return errors
|
||||
|
||||
# We need a subject in case domain wide delegation is set, otherwise the
|
||||
|
@ -110,17 +172,27 @@ class GSheetsEngineSpec(SqliteEngineSpec):
|
|||
)
|
||||
conn = engine.connect()
|
||||
for name, url in table_catalog.items():
|
||||
|
||||
if not name:
|
||||
errors.append(
|
||||
SupersetError(
|
||||
message="Sheet name is required",
|
||||
error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={"invalid": [], "name": name, "url": url},
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
results = conn.execute(f'SELECT * FROM "{url}" LIMIT 1')
|
||||
results.fetchall()
|
||||
except Exception: # pylint: disable=broad-except
|
||||
errors.append(
|
||||
SupersetError(
|
||||
message=f"Unable to connect to spreadsheet {name} at {url}",
|
||||
message="URL could not be identified",
|
||||
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={"name": name, "url": url},
|
||||
extra={"invalid": ["catalog"], "name": name, "url": url},
|
||||
),
|
||||
)
|
||||
|
||||
return errors
|
||||
|
|
|
@ -39,6 +39,7 @@ from superset.db_engine_specs.mysql import MySQLEngineSpec
|
|||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
from superset.db_engine_specs.bigquery import BigQueryEngineSpec
|
||||
from superset.db_engine_specs.gsheets import GSheetsEngineSpec
|
||||
from superset.db_engine_specs.hana import HanaEngineSpec
|
||||
from superset.errors import SupersetError
|
||||
from superset.models.core import Database, ConfigurationMethod
|
||||
|
@ -1438,6 +1439,7 @@ class TestDatabaseApi(SupersetTestCase):
|
|||
PostgresEngineSpec: {"psycopg2"},
|
||||
BigQueryEngineSpec: {"bigquery"},
|
||||
MySQLEngineSpec: {"mysqlconnector", "mysqldb"},
|
||||
GSheetsEngineSpec: {"apsw"},
|
||||
RedshiftEngineSpec: {"psycopg2"},
|
||||
HanaEngineSpec: {""},
|
||||
}
|
||||
|
@ -1565,6 +1567,18 @@ class TestDatabaseApi(SupersetTestCase):
|
|||
"preferred": False,
|
||||
"sqlalchemy_uri_placeholder": "redshift+psycopg2://user:password@host:port/dbname[?key=value&key=value...]",
|
||||
},
|
||||
{
|
||||
"available_drivers": ["apsw"],
|
||||
"default_driver": "apsw",
|
||||
"engine": "gsheets",
|
||||
"name": "Google Sheets",
|
||||
"parameters": {
|
||||
"properties": {"catalog": {"type": "object"},},
|
||||
"type": "object",
|
||||
},
|
||||
"preferred": False,
|
||||
"sqlalchemy_uri_placeholder": "gsheets://",
|
||||
},
|
||||
{
|
||||
"available_drivers": ["mysqlconnector", "mysqldb"],
|
||||
"default_driver": "mysqldb",
|
||||
|
|
|
@ -37,11 +37,27 @@ def test_validate_parameters_simple(
|
|||
|
||||
parameters: GSheetsParametersType = {
|
||||
"credentials_info": {},
|
||||
"query": {},
|
||||
"table_catalog": {},
|
||||
"catalog": {},
|
||||
}
|
||||
errors = GSheetsEngineSpec.validate_parameters(parameters)
|
||||
assert errors == []
|
||||
assert errors == [
|
||||
SupersetError(
|
||||
message="URL is required",
|
||||
error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={
|
||||
"invalid": ["catalog"],
|
||||
"name": "",
|
||||
"url": "",
|
||||
"issue_codes": [
|
||||
{
|
||||
"code": 1018,
|
||||
"message": "Issue 1018 - One or more parameters needed to configure a database are missing.",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_validate_parameters_catalog(
|
||||
|
@ -66,72 +82,57 @@ def test_validate_parameters_catalog(
|
|||
|
||||
parameters: GSheetsParametersType = {
|
||||
"credentials_info": {},
|
||||
"query": {},
|
||||
"table_catalog": {
|
||||
"catalog": {
|
||||
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
|
||||
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
|
||||
"not_a_sheet": "https://www.google.com/",
|
||||
},
|
||||
}
|
||||
errors = GSheetsEngineSpec.validate_parameters(parameters)
|
||||
errors = GSheetsEngineSpec.validate_parameters(parameters) # ignore: type
|
||||
|
||||
assert errors == [
|
||||
SupersetError(
|
||||
message=(
|
||||
"Unable to connect to spreadsheet private_sheet at "
|
||||
"https://docs.google.com/spreadsheets/d/1/edit"
|
||||
),
|
||||
message="URL could not be identified",
|
||||
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={
|
||||
"invalid": ["catalog"],
|
||||
"name": "private_sheet",
|
||||
"url": "https://docs.google.com/spreadsheets/d/1/edit",
|
||||
"issue_codes": [
|
||||
{
|
||||
"code": 1003,
|
||||
"message": (
|
||||
"Issue 1003 - There is a syntax error in the SQL query. "
|
||||
"Perhaps there was a misspelling or a typo."
|
||||
),
|
||||
"message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.",
|
||||
},
|
||||
{
|
||||
"code": 1005,
|
||||
"message": (
|
||||
"Issue 1005 - The table was deleted or renamed in the "
|
||||
"database."
|
||||
),
|
||||
"message": "Issue 1005 - The table was deleted or renamed in the database.",
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
SupersetError(
|
||||
message=(
|
||||
"Unable to connect to spreadsheet not_a_sheet at "
|
||||
"https://www.google.com/"
|
||||
),
|
||||
message="URL could not be identified",
|
||||
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={
|
||||
"invalid": ["catalog"],
|
||||
"name": "not_a_sheet",
|
||||
"url": "https://www.google.com/",
|
||||
"issue_codes": [
|
||||
{
|
||||
"code": 1003,
|
||||
"message": (
|
||||
"Issue 1003 - There is a syntax error in the SQL query. "
|
||||
"Perhaps there was a misspelling or a typo."
|
||||
),
|
||||
"message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.",
|
||||
},
|
||||
{
|
||||
"code": 1005,
|
||||
"message": (
|
||||
"Issue 1005 - The table was deleted or renamed in the "
|
||||
"database.",
|
||||
),
|
||||
"message": "Issue 1005 - The table was deleted or renamed in the database.",
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
create_engine.assert_called_with(
|
||||
"gsheets://", service_account_info={}, subject="admin@example.com",
|
||||
)
|
||||
|
@ -159,44 +160,36 @@ def test_validate_parameters_catalog_and_credentials(
|
|||
|
||||
parameters: GSheetsParametersType = {
|
||||
"credentials_info": {},
|
||||
"query": {},
|
||||
"table_catalog": {
|
||||
"catalog": {
|
||||
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
|
||||
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
|
||||
"not_a_sheet": "https://www.google.com/",
|
||||
},
|
||||
}
|
||||
errors = GSheetsEngineSpec.validate_parameters(parameters)
|
||||
errors = GSheetsEngineSpec.validate_parameters(parameters) # ignore: type
|
||||
assert errors == [
|
||||
SupersetError(
|
||||
message=(
|
||||
"Unable to connect to spreadsheet not_a_sheet at "
|
||||
"https://www.google.com/"
|
||||
),
|
||||
message="URL could not be identified",
|
||||
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
|
||||
level=ErrorLevel.WARNING,
|
||||
extra={
|
||||
"invalid": ["catalog"],
|
||||
"name": "not_a_sheet",
|
||||
"url": "https://www.google.com/",
|
||||
"issue_codes": [
|
||||
{
|
||||
"code": 1003,
|
||||
"message": (
|
||||
"Issue 1003 - There is a syntax error in the SQL query. "
|
||||
"Perhaps there was a misspelling or a typo."
|
||||
),
|
||||
"message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.",
|
||||
},
|
||||
{
|
||||
"code": 1005,
|
||||
"message": (
|
||||
"Issue 1005 - The table was deleted or renamed in the "
|
||||
"database.",
|
||||
),
|
||||
"message": "Issue 1005 - The table was deleted or renamed in the database.",
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
create_engine.assert_called_with(
|
||||
"gsheets://", service_account_info={}, subject="admin@example.com",
|
||||
)
|
||||
|
|
Loading…
Reference in New Issue