mirror of https://github.com/apache/superset.git
feat: Add Private Google Sheets to dynamic form (#16628)
* first pass private gsheets * made encrypted extra into string, refactored onParametersChanged * private sheets working, credential_info errors * all but test connection working * first pass private gsheets * made encrypted extra into string, refactored onParametersChanged * private sheets working, credential_info errors * all but test connection working * Regenerate package-lock.json Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>
This commit is contained in:
parent
b35645c3f4
commit
aa747219ad
File diff suppressed because it is too large
Load Diff
|
@ -32,14 +32,25 @@ import {
|
||||||
infoTooltip,
|
infoTooltip,
|
||||||
StyledFooterButton,
|
StyledFooterButton,
|
||||||
StyledCatalogTable,
|
StyledCatalogTable,
|
||||||
|
labelMarginBotton,
|
||||||
} from './styles';
|
} from './styles';
|
||||||
import { CatalogObject, DatabaseForm, DatabaseObject } from '../types';
|
import { CatalogObject, DatabaseForm, DatabaseObject } from '../types';
|
||||||
|
|
||||||
|
// These are the columns that are going to be added to encrypted extra, they differ in name based
|
||||||
|
// on the engine, however we want to use the same component for each of them. Make sure to add the
|
||||||
|
// the engine specific name here.
|
||||||
|
export const encryptedCredentialsMap = {
|
||||||
|
gsheets: 'service_account_info',
|
||||||
|
bigquery: 'credentials_info',
|
||||||
|
};
|
||||||
|
|
||||||
enum CredentialInfoOptions {
|
enum CredentialInfoOptions {
|
||||||
jsonUpload,
|
jsonUpload,
|
||||||
copyPaste,
|
copyPaste,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const castStringToBoolean = (optionValue: string) => optionValue === 'true';
|
||||||
|
|
||||||
export const FormFieldOrder = [
|
export const FormFieldOrder = [
|
||||||
'host',
|
'host',
|
||||||
'port',
|
'port',
|
||||||
|
@ -48,6 +59,7 @@ export const FormFieldOrder = [
|
||||||
'password',
|
'password',
|
||||||
'database_name',
|
'database_name',
|
||||||
'credentials_info',
|
'credentials_info',
|
||||||
|
'service_account_info',
|
||||||
'catalog',
|
'catalog',
|
||||||
'query',
|
'query',
|
||||||
'encryption',
|
'encryption',
|
||||||
|
@ -56,7 +68,7 @@ export const FormFieldOrder = [
|
||||||
interface FieldPropTypes {
|
interface FieldPropTypes {
|
||||||
required: boolean;
|
required: boolean;
|
||||||
hasTooltip?: boolean;
|
hasTooltip?: boolean;
|
||||||
tooltipText?: (valuse: any) => string;
|
tooltipText?: (value: any) => string;
|
||||||
onParametersChange: (value: any) => string;
|
onParametersChange: (value: any) => string;
|
||||||
onParametersUploadFileChange: (value: any) => string;
|
onParametersUploadFileChange: (value: any) => string;
|
||||||
changeMethods: { onParametersChange: (value: any) => string } & {
|
changeMethods: { onParametersChange: (value: any) => string } & {
|
||||||
|
@ -88,12 +100,46 @@ const CredentialsInfo = ({
|
||||||
const [fileToUpload, setFileToUpload] = useState<string | null | undefined>(
|
const [fileToUpload, setFileToUpload] = useState<string | null | undefined>(
|
||||||
null,
|
null,
|
||||||
);
|
);
|
||||||
|
const [isPublic, setIsPublic] = useState<boolean>(true);
|
||||||
|
const showCredentialsInfo =
|
||||||
|
db?.engine === 'gsheets' ? !isEditMode && !isPublic : !isEditMode;
|
||||||
|
// a database that has an optional encrypted field has an encrypted_extra that is an empty object, this checks for that.
|
||||||
|
const isEncrypted = isEditMode && db?.encrypted_extra !== '{}';
|
||||||
|
const encryptedField = db?.engine && encryptedCredentialsMap[db.engine];
|
||||||
|
const encryptedValue =
|
||||||
|
typeof db?.parameters?.[encryptedField] === 'object'
|
||||||
|
? JSON.stringify(db?.parameters?.[encryptedField])
|
||||||
|
: db?.parameters?.[encryptedField];
|
||||||
return (
|
return (
|
||||||
<CredentialInfoForm>
|
<CredentialInfoForm>
|
||||||
{!isEditMode && (
|
{db?.engine === 'gsheets' && (
|
||||||
|
<div className="catalog-type-select">
|
||||||
|
<FormLabel
|
||||||
|
css={(theme: SupersetTheme) => labelMarginBotton(theme)}
|
||||||
|
required
|
||||||
|
>
|
||||||
|
{t('Type of Google Sheets allowed')}
|
||||||
|
</FormLabel>
|
||||||
|
<Select
|
||||||
|
style={{ width: '100%' }}
|
||||||
|
defaultValue={isEncrypted ? 'false' : 'true'}
|
||||||
|
onChange={(value: string) =>
|
||||||
|
setIsPublic(castStringToBoolean(value))
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<Select.Option value="true" key={1}>
|
||||||
|
{t('Publicly shared sheets only')}
|
||||||
|
</Select.Option>
|
||||||
|
<Select.Option value="false" key={2}>
|
||||||
|
{t('Public and privately shared sheets')}
|
||||||
|
</Select.Option>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{showCredentialsInfo && (
|
||||||
<>
|
<>
|
||||||
<FormLabel required>
|
<FormLabel required>
|
||||||
{t('How do you want to enter service account credentials?')}
|
{t('How∂ do you want to enter service account credentials?')}
|
||||||
</FormLabel>
|
</FormLabel>
|
||||||
<Select
|
<Select
|
||||||
defaultValue={uploadOption}
|
defaultValue={uploadOption}
|
||||||
|
@ -117,8 +163,8 @@ const CredentialsInfo = ({
|
||||||
<FormLabel required>{t('Service Account')}</FormLabel>
|
<FormLabel required>{t('Service Account')}</FormLabel>
|
||||||
<textarea
|
<textarea
|
||||||
className="input-form"
|
className="input-form"
|
||||||
name="credentials_info"
|
name={encryptedField}
|
||||||
value={db?.parameters?.credentials_info}
|
value={encryptedValue}
|
||||||
onChange={changeMethods.onParametersChange}
|
onChange={changeMethods.onParametersChange}
|
||||||
placeholder="Paste content of service credentials JSON file here"
|
placeholder="Paste content of service credentials JSON file here"
|
||||||
/>
|
/>
|
||||||
|
@ -127,6 +173,7 @@ const CredentialsInfo = ({
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
|
showCredentialsInfo && (
|
||||||
<div
|
<div
|
||||||
className="input-container"
|
className="input-container"
|
||||||
css={(theme: SupersetTheme) => infoTooltip(theme)}
|
css={(theme: SupersetTheme) => infoTooltip(theme)}
|
||||||
|
@ -135,7 +182,7 @@ const CredentialsInfo = ({
|
||||||
<FormLabel required>{t('Upload Credentials')}</FormLabel>
|
<FormLabel required>{t('Upload Credentials')}</FormLabel>
|
||||||
<InfoTooltip
|
<InfoTooltip
|
||||||
tooltip={t(
|
tooltip={t(
|
||||||
'Use the JSON file you automatically downloaded when creating your service account in Google BigQuery.',
|
'Use the JSON file you automatically downloaded when creating your service account.',
|
||||||
)}
|
)}
|
||||||
viewBox="0 0 24 24"
|
viewBox="0 0 24 24"
|
||||||
/>
|
/>
|
||||||
|
@ -144,7 +191,9 @@ const CredentialsInfo = ({
|
||||||
{!fileToUpload && (
|
{!fileToUpload && (
|
||||||
<Button
|
<Button
|
||||||
className="input-upload-btn"
|
className="input-upload-btn"
|
||||||
onClick={() => document?.getElementById('selectedFile')?.click()}
|
onClick={() =>
|
||||||
|
document?.getElementById('selectedFile')?.click()
|
||||||
|
}
|
||||||
>
|
>
|
||||||
{t('Choose File')}
|
{t('Choose File')}
|
||||||
</Button>
|
</Button>
|
||||||
|
@ -157,7 +206,7 @@ const CredentialsInfo = ({
|
||||||
setFileToUpload(null);
|
setFileToUpload(null);
|
||||||
changeMethods.onParametersChange({
|
changeMethods.onParametersChange({
|
||||||
target: {
|
target: {
|
||||||
name: 'credentials_info',
|
name: encryptedField,
|
||||||
value: '',
|
value: '',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
@ -179,7 +228,7 @@ const CredentialsInfo = ({
|
||||||
changeMethods.onParametersChange({
|
changeMethods.onParametersChange({
|
||||||
target: {
|
target: {
|
||||||
type: null,
|
type: null,
|
||||||
name: 'credentials_info',
|
name: encryptedField,
|
||||||
value: await file?.text(),
|
value: await file?.text(),
|
||||||
checked: false,
|
checked: false,
|
||||||
},
|
},
|
||||||
|
@ -190,6 +239,7 @@ const CredentialsInfo = ({
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
)
|
||||||
)}
|
)}
|
||||||
</CredentialInfoForm>
|
</CredentialInfoForm>
|
||||||
);
|
);
|
||||||
|
@ -204,16 +254,9 @@ const TableCatalog = ({
|
||||||
}: FieldPropTypes) => {
|
}: FieldPropTypes) => {
|
||||||
const tableCatalog = db?.catalog || [];
|
const tableCatalog = db?.catalog || [];
|
||||||
const catalogError = validationErrors || {};
|
const catalogError = validationErrors || {};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<StyledCatalogTable>
|
<StyledCatalogTable>
|
||||||
<div className="catalog-type-select">
|
|
||||||
<FormLabel required>{t('Type of Google Sheets Allowed')}</FormLabel>
|
|
||||||
<Select style={{ width: '100%' }} defaultValue="true" disabled>
|
|
||||||
<Select.Option value="true" key={1}>
|
|
||||||
{t('Publicly shared sheets only')}
|
|
||||||
</Select.Option>
|
|
||||||
</Select>
|
|
||||||
</div>
|
|
||||||
<h4 className="gsheet-title">
|
<h4 className="gsheet-title">
|
||||||
{t('Connect Google Sheets as tables to this database')}
|
{t('Connect Google Sheets as tables to this database')}
|
||||||
</h4>
|
</h4>
|
||||||
|
@ -472,6 +515,7 @@ const FORM_FIELD_MAP = {
|
||||||
query: queryField,
|
query: queryField,
|
||||||
encryption: forceSSLField,
|
encryption: forceSSLField,
|
||||||
credentials_info: CredentialsInfo,
|
credentials_info: CredentialsInfo,
|
||||||
|
service_account_info: CredentialsInfo,
|
||||||
catalog: TableCatalog,
|
catalog: TableCatalog,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -180,6 +180,7 @@ type DBReducerActionType =
|
||||||
database_name?: string;
|
database_name?: string;
|
||||||
engine?: string;
|
engine?: string;
|
||||||
configuration_method: CONFIGURATION_METHOD;
|
configuration_method: CONFIGURATION_METHOD;
|
||||||
|
paramProperties?: Record<string, any>;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
|
@ -358,47 +359,27 @@ function dbReducer(
|
||||||
.join('&');
|
.join('&');
|
||||||
|
|
||||||
if (
|
if (
|
||||||
action.payload.backend === 'bigquery' &&
|
action.payload.encrypted_extra &&
|
||||||
action.payload.configuration_method ===
|
action.payload.configuration_method ===
|
||||||
CONFIGURATION_METHOD.DYNAMIC_FORM
|
CONFIGURATION_METHOD.DYNAMIC_FORM
|
||||||
) {
|
) {
|
||||||
|
const engineParamsCatalog = Object.keys(
|
||||||
|
extra_json?.engine_params?.catalog || {},
|
||||||
|
).map(e => ({
|
||||||
|
name: e,
|
||||||
|
value: extra_json?.engine_params?.catalog[e],
|
||||||
|
}));
|
||||||
return {
|
return {
|
||||||
...action.payload,
|
...action.payload,
|
||||||
engine: action.payload.backend,
|
engine: action.payload.backend || trimmedState.engine,
|
||||||
configuration_method: action.payload.configuration_method,
|
configuration_method: action.payload.configuration_method,
|
||||||
extra_json: deserializeExtraJSON,
|
extra_json: deserializeExtraJSON,
|
||||||
parameters: {
|
catalog: engineParamsCatalog,
|
||||||
credentials_info: JSON.stringify(
|
parameters: action.payload.parameters,
|
||||||
action.payload?.parameters?.credentials_info || '',
|
|
||||||
),
|
|
||||||
query,
|
|
||||||
},
|
|
||||||
query_input,
|
query_input,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
|
||||||
action.payload.backend === 'gsheets' &&
|
|
||||||
action.payload.configuration_method ===
|
|
||||||
CONFIGURATION_METHOD.DYNAMIC_FORM &&
|
|
||||||
extra_json?.engine_params?.catalog !== undefined
|
|
||||||
) {
|
|
||||||
// pull catalog from engine params
|
|
||||||
const engineParamsCatalog = extra_json?.engine_params?.catalog;
|
|
||||||
|
|
||||||
return {
|
|
||||||
...action.payload,
|
|
||||||
engine: action.payload.backend,
|
|
||||||
configuration_method: action.payload.configuration_method,
|
|
||||||
extra_json: deserializeExtraJSON,
|
|
||||||
catalog: Object.keys(engineParamsCatalog).map(e => ({
|
|
||||||
name: e,
|
|
||||||
value: engineParamsCatalog[e],
|
|
||||||
})),
|
|
||||||
query_input,
|
|
||||||
} as DatabaseObject;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
...action.payload,
|
...action.payload,
|
||||||
encrypted_extra: action.payload.encrypted_extra || '',
|
encrypted_extra: action.payload.encrypted_extra || '',
|
||||||
|
@ -513,7 +494,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
||||||
encrypted_extra: db?.encrypted_extra || '',
|
encrypted_extra: db?.encrypted_extra || '',
|
||||||
server_cert: db?.server_cert || undefined,
|
server_cert: db?.server_cert || undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
testDatabaseConnection(connection, addDangerToast, addSuccessToast);
|
testDatabaseConnection(connection, addDangerToast, addSuccessToast);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -525,11 +505,9 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
||||||
setEditNewDb(false);
|
setEditNewDb(false);
|
||||||
onHide();
|
onHide();
|
||||||
};
|
};
|
||||||
|
|
||||||
const onSave = async () => {
|
const onSave = async () => {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
const { id, ...update } = db || {};
|
const { id, ...update } = db || {};
|
||||||
|
|
||||||
// Clone DB object
|
// Clone DB object
|
||||||
const dbToUpdate = JSON.parse(JSON.stringify(update));
|
const dbToUpdate = JSON.parse(JSON.stringify(update));
|
||||||
|
|
||||||
|
@ -539,32 +517,41 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
||||||
if (validationErrors && !isEmpty(validationErrors)) {
|
if (validationErrors && !isEmpty(validationErrors)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const parameters_schema = isEditMode
|
||||||
|
? dbToUpdate.parameters_schema.properties
|
||||||
|
: dbModel?.parameters.properties;
|
||||||
|
const additionalEncryptedExtra = JSON.parse(
|
||||||
|
dbToUpdate.encrypted_extra || '{}',
|
||||||
|
);
|
||||||
|
const paramConfigArray = Object.keys(parameters_schema || {});
|
||||||
|
|
||||||
const engine = dbToUpdate.backend || dbToUpdate.engine;
|
paramConfigArray.forEach(paramConfig => {
|
||||||
if (engine === 'bigquery' && dbToUpdate.parameters?.credentials_info) {
|
/*
|
||||||
// wrap encrypted_extra in credentials_info only for BigQuery
|
* Parameters that are annotated with the `x-encrypted-extra` properties should be moved to
|
||||||
|
* `encrypted_extra`, so that they are stored encrypted in the backend when the database is
|
||||||
|
* created or edited.
|
||||||
|
*/
|
||||||
if (
|
if (
|
||||||
dbToUpdate.parameters?.credentials_info &&
|
parameters_schema[paramConfig]['x-encrypted-extra'] &&
|
||||||
typeof dbToUpdate.parameters?.credentials_info === 'object' &&
|
dbToUpdate.parameters?.[paramConfig]
|
||||||
dbToUpdate.parameters?.credentials_info.constructor === Object
|
|
||||||
) {
|
) {
|
||||||
// Don't cast if object
|
if (typeof dbToUpdate.parameters?.[paramConfig] === 'object') {
|
||||||
dbToUpdate.encrypted_extra = JSON.stringify({
|
// add new encrypted extra to encrypted_extra object
|
||||||
credentials_info: dbToUpdate.parameters?.credentials_info,
|
additionalEncryptedExtra[paramConfig] =
|
||||||
});
|
dbToUpdate.parameters?.[paramConfig];
|
||||||
|
// The backend expects `encrypted_extra` as a string for historical reasons.
|
||||||
// Convert credentials info string before updating
|
dbToUpdate.parameters[paramConfig] = JSON.stringify(
|
||||||
dbToUpdate.parameters.credentials_info = JSON.stringify(
|
dbToUpdate.parameters[paramConfig],
|
||||||
dbToUpdate.parameters.credentials_info,
|
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
dbToUpdate.encrypted_extra = JSON.stringify({
|
additionalEncryptedExtra[paramConfig] = JSON.parse(
|
||||||
credentials_info: JSON.parse(
|
dbToUpdate.parameters?.[paramConfig] || '{}',
|
||||||
dbToUpdate.parameters?.credentials_info,
|
);
|
||||||
),
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
// cast the new encrypted extra object into a string
|
||||||
}
|
dbToUpdate.encrypted_extra = JSON.stringify(additionalEncryptedExtra);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbToUpdate?.parameters?.catalog) {
|
if (dbToUpdate?.parameters?.catalog) {
|
||||||
|
@ -660,10 +647,11 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
|
||||||
type: ActionType.dbSelected,
|
type: ActionType.dbSelected,
|
||||||
payload: {
|
payload: {
|
||||||
database_name,
|
database_name,
|
||||||
|
engine,
|
||||||
configuration_method: isDynamic
|
configuration_method: isDynamic
|
||||||
? CONFIGURATION_METHOD.DYNAMIC_FORM
|
? CONFIGURATION_METHOD.DYNAMIC_FORM
|
||||||
: CONFIGURATION_METHOD.SQLALCHEMY_URI,
|
: CONFIGURATION_METHOD.SQLALCHEMY_URI,
|
||||||
engine,
|
paramProperties: parameters?.properties,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,6 +31,10 @@ export const no_margin_bottom = css`
|
||||||
margin-bottom: 0;
|
margin-bottom: 0;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
export const labelMarginBotton = (theme: SupersetTheme) => css`
|
||||||
|
margin-bottom: ${theme.gridUnit * 2}px;
|
||||||
|
`;
|
||||||
|
|
||||||
export const marginBottom = (theme: SupersetTheme) => css`
|
export const marginBottom = (theme: SupersetTheme) => css`
|
||||||
margin-bottom: ${theme.gridUnit * 4}px;
|
margin-bottom: ${theme.gridUnit * 4}px;
|
||||||
`;
|
`;
|
||||||
|
@ -442,6 +446,10 @@ export const EditHeaderSubtitle = styled.div`
|
||||||
`;
|
`;
|
||||||
|
|
||||||
export const CredentialInfoForm = styled.div`
|
export const CredentialInfoForm = styled.div`
|
||||||
|
.catalog-type-select {
|
||||||
|
margin: 0 0 20px;
|
||||||
|
}
|
||||||
|
|
||||||
.label-select {
|
.label-select {
|
||||||
text-transform: uppercase;
|
text-transform: uppercase;
|
||||||
color: ${({ theme }) => theme.colors.grayscale.dark1};
|
color: ${({ theme }) => theme.colors.grayscale.dark1};
|
||||||
|
@ -542,13 +550,13 @@ export const StyledCatalogTable = styled.div`
|
||||||
margin-bottom: 16px;
|
margin-bottom: 16px;
|
||||||
|
|
||||||
.catalog-type-select {
|
.catalog-type-select {
|
||||||
margin: 0 0 40px;
|
margin: 0 0 20px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.gsheet-title {
|
.gsheet-title {
|
||||||
font-size: ${({ theme }) => theme.typography.sizes.l * 1.1}px;
|
font-size: ${({ theme }) => theme.typography.sizes.l * 1.1}px;
|
||||||
font-weight: bold;
|
font-weight: bold;
|
||||||
margin: ${({ theme }) => theme.gridUnit * 6}px 0 16px;
|
margin: ${({ theme }) => theme.gridUnit * 10}px 0 16px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.catalog-label {
|
.catalog-label {
|
||||||
|
|
|
@ -45,11 +45,14 @@ export type DatabaseObject = {
|
||||||
password?: string;
|
password?: string;
|
||||||
encryption?: boolean;
|
encryption?: boolean;
|
||||||
credentials_info?: string;
|
credentials_info?: string;
|
||||||
|
service_account_info?: string;
|
||||||
query?: Record<string, string>;
|
query?: Record<string, string>;
|
||||||
catalog?: Record<string, string>;
|
catalog?: Record<string, string>;
|
||||||
|
properties?: Record<string, any>;
|
||||||
};
|
};
|
||||||
configuration_method: CONFIGURATION_METHOD;
|
configuration_method: CONFIGURATION_METHOD;
|
||||||
engine?: string;
|
engine?: string;
|
||||||
|
paramProperties?: Record<string, any>;
|
||||||
|
|
||||||
// Performance
|
// Performance
|
||||||
cache_timeout?: string;
|
cache_timeout?: string;
|
||||||
|
@ -68,6 +71,7 @@ export type DatabaseObject = {
|
||||||
server_cert?: string;
|
server_cert?: string;
|
||||||
allow_csv_upload?: boolean;
|
allow_csv_upload?: boolean;
|
||||||
impersonate_user?: boolean;
|
impersonate_user?: boolean;
|
||||||
|
parameters_schema?: Record<string, any>;
|
||||||
|
|
||||||
// Extra
|
// Extra
|
||||||
extra_json?: {
|
extra_json?: {
|
||||||
|
@ -131,6 +135,11 @@ export type DatabaseForm = {
|
||||||
nullable: boolean;
|
nullable: boolean;
|
||||||
type: string;
|
type: string;
|
||||||
};
|
};
|
||||||
|
service_account_info: {
|
||||||
|
description: string;
|
||||||
|
nullable: boolean;
|
||||||
|
type: string;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
required: string[];
|
required: string[];
|
||||||
type: string;
|
type: string;
|
||||||
|
|
|
@ -116,6 +116,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||||
"encrypted_extra",
|
"encrypted_extra",
|
||||||
"extra",
|
"extra",
|
||||||
"parameters",
|
"parameters",
|
||||||
|
"parameters_schema",
|
||||||
"server_cert",
|
"server_cert",
|
||||||
"sqlalchemy_uri",
|
"sqlalchemy_uri",
|
||||||
]
|
]
|
||||||
|
|
|
@ -602,7 +602,17 @@ class ImportV1DatabaseSchema(Schema):
|
||||||
raise ValidationError("Must provide a password for the database")
|
raise ValidationError("Must provide a password for the database")
|
||||||
|
|
||||||
|
|
||||||
class EncryptedField(fields.String):
|
class EncryptedField: # pylint: disable=too-few-public-methods
|
||||||
|
"""
|
||||||
|
A database field that should be stored in encrypted_extra.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class EncryptedString(EncryptedField, fields.String):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class EncryptedDict(EncryptedField, fields.Dict):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
# KIND, either express or implied. See the License for the
|
# KIND, either express or implied. See the License for the
|
||||||
# specific language governing permissions and limitations
|
# specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
import json
|
||||||
import re
|
import re
|
||||||
import urllib
|
import urllib
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
@ -30,7 +31,7 @@ from sqlalchemy.engine.url import make_url
|
||||||
from sqlalchemy.sql.expression import ColumnClause
|
from sqlalchemy.sql.expression import ColumnClause
|
||||||
from typing_extensions import TypedDict
|
from typing_extensions import TypedDict
|
||||||
|
|
||||||
from superset.databases.schemas import encrypted_field_properties, EncryptedField
|
from superset.databases.schemas import encrypted_field_properties, EncryptedString
|
||||||
from superset.db_engine_specs.base import BaseEngineSpec
|
from superset.db_engine_specs.base import BaseEngineSpec
|
||||||
from superset.db_engine_specs.exceptions import SupersetDBAPIDisconnectionError
|
from superset.db_engine_specs.exceptions import SupersetDBAPIDisconnectionError
|
||||||
from superset.errors import SupersetError, SupersetErrorType
|
from superset.errors import SupersetError, SupersetErrorType
|
||||||
|
@ -69,7 +70,7 @@ ma_plugin = MarshmallowPlugin()
|
||||||
|
|
||||||
|
|
||||||
class BigQueryParametersSchema(Schema):
|
class BigQueryParametersSchema(Schema):
|
||||||
credentials_info = EncryptedField(
|
credentials_info = EncryptedString(
|
||||||
required=False, description="Contents of BigQuery JSON credentials.",
|
required=False, description="Contents of BigQuery JSON credentials.",
|
||||||
)
|
)
|
||||||
query = fields.Dict(required=False)
|
query = fields.Dict(required=False)
|
||||||
|
@ -367,11 +368,14 @@ class BigQueryEngineSpec(BaseEngineSpec):
|
||||||
query = parameters.get("query", {})
|
query = parameters.get("query", {})
|
||||||
query_params = urllib.parse.urlencode(query)
|
query_params = urllib.parse.urlencode(query)
|
||||||
|
|
||||||
|
if encrypted_extra:
|
||||||
|
credentials_info = encrypted_extra.get("credentials_info")
|
||||||
|
if isinstance(credentials_info, str):
|
||||||
|
credentials_info = json.loads(credentials_info)
|
||||||
|
project_id = credentials_info.get("project_id")
|
||||||
if not encrypted_extra:
|
if not encrypted_extra:
|
||||||
raise ValidationError("Missing service credentials")
|
raise ValidationError("Missing service credentials")
|
||||||
|
|
||||||
project_id = encrypted_extra.get("credentials_info", {}).get("project_id")
|
|
||||||
|
|
||||||
if project_id:
|
if project_id:
|
||||||
return f"{cls.default_driver}://{project_id}/?{query_params}"
|
return f"{cls.default_driver}://{project_id}/?{query_params}"
|
||||||
|
|
||||||
|
|
|
@ -30,7 +30,7 @@ from sqlalchemy.engine.url import URL
|
||||||
from typing_extensions import TypedDict
|
from typing_extensions import TypedDict
|
||||||
|
|
||||||
from superset import security_manager
|
from superset import security_manager
|
||||||
from superset.databases.schemas import encrypted_field_properties
|
from superset.databases.schemas import encrypted_field_properties, EncryptedString
|
||||||
from superset.db_engine_specs.sqlite import SqliteEngineSpec
|
from superset.db_engine_specs.sqlite import SqliteEngineSpec
|
||||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||||
|
|
||||||
|
@ -45,10 +45,15 @@ ma_plugin = MarshmallowPlugin()
|
||||||
|
|
||||||
class GSheetsParametersSchema(Schema):
|
class GSheetsParametersSchema(Schema):
|
||||||
catalog = fields.Dict()
|
catalog = fields.Dict()
|
||||||
|
service_account_info = EncryptedString(
|
||||||
|
required=False,
|
||||||
|
description="Contents of GSheets JSON credentials.",
|
||||||
|
field_name="service_account_info",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class GSheetsParametersType(TypedDict):
|
class GSheetsParametersType(TypedDict):
|
||||||
credentials_info: Dict[str, Any]
|
service_account_info: str
|
||||||
catalog: Dict[str, str]
|
catalog: Dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
|
@ -113,7 +118,9 @@ class GSheetsEngineSpec(SqliteEngineSpec):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_parameters_from_uri(
|
def get_parameters_from_uri(
|
||||||
cls, encrypted_extra: Optional[Dict[str, str]] = None,
|
cls,
|
||||||
|
uri: str, # pylint: disable=unused-argument
|
||||||
|
encrypted_extra: Optional[Dict[str, str]] = None,
|
||||||
) -> Any:
|
) -> Any:
|
||||||
# Building parameters from encrypted_extra and uri
|
# Building parameters from encrypted_extra and uri
|
||||||
if encrypted_extra:
|
if encrypted_extra:
|
||||||
|
@ -146,7 +153,10 @@ class GSheetsEngineSpec(SqliteEngineSpec):
|
||||||
cls, parameters: GSheetsParametersType,
|
cls, parameters: GSheetsParametersType,
|
||||||
) -> List[SupersetError]:
|
) -> List[SupersetError]:
|
||||||
errors: List[SupersetError] = []
|
errors: List[SupersetError] = []
|
||||||
credentials_info = parameters.get("credentials_info")
|
encrypted_credentials = json.loads(
|
||||||
|
parameters.get("service_account_info") or "{}"
|
||||||
|
)
|
||||||
|
|
||||||
table_catalog = parameters.get("catalog", {})
|
table_catalog = parameters.get("catalog", {})
|
||||||
|
|
||||||
if not table_catalog:
|
if not table_catalog:
|
||||||
|
@ -160,7 +170,7 @@ class GSheetsEngineSpec(SqliteEngineSpec):
|
||||||
subject = g.user.email if g.user else None
|
subject = g.user.email if g.user else None
|
||||||
|
|
||||||
engine = create_engine(
|
engine = create_engine(
|
||||||
"gsheets://", service_account_info=credentials_info, subject=subject,
|
"gsheets://", service_account_info=encrypted_credentials, subject=subject,
|
||||||
)
|
)
|
||||||
conn = engine.connect()
|
conn = engine.connect()
|
||||||
idx = 0
|
idx = 0
|
||||||
|
|
|
@ -223,6 +223,7 @@ class Database(
|
||||||
"allows_virtual_table_explore": self.allows_virtual_table_explore,
|
"allows_virtual_table_explore": self.allows_virtual_table_explore,
|
||||||
"explore_database_id": self.explore_database_id,
|
"explore_database_id": self.explore_database_id,
|
||||||
"parameters": self.parameters,
|
"parameters": self.parameters,
|
||||||
|
"parameters_schema": self.parameters_schema,
|
||||||
}
|
}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
@ -249,6 +250,14 @@ class Database(
|
||||||
|
|
||||||
return parameters
|
return parameters
|
||||||
|
|
||||||
|
@property
|
||||||
|
def parameters_schema(self) -> Dict[str, Any]:
|
||||||
|
try:
|
||||||
|
parameters_schema = self.db_engine_spec.parameters_json_schema() # type: ignore # pylint: disable=line-too-long
|
||||||
|
except Exception: # pylint: disable=broad-except
|
||||||
|
parameters_schema = {}
|
||||||
|
return parameters_schema
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def metadata_cache_timeout(self) -> Dict[str, Any]:
|
def metadata_cache_timeout(self) -> Dict[str, Any]:
|
||||||
return self.get_extra().get("metadata_cache_timeout", {})
|
return self.get_extra().get("metadata_cache_timeout", {})
|
||||||
|
|
|
@ -1576,7 +1576,14 @@ class TestDatabaseApi(SupersetTestCase):
|
||||||
"engine": "gsheets",
|
"engine": "gsheets",
|
||||||
"name": "Google Sheets",
|
"name": "Google Sheets",
|
||||||
"parameters": {
|
"parameters": {
|
||||||
"properties": {"catalog": {"type": "object"},},
|
"properties": {
|
||||||
|
"catalog": {"type": "object"},
|
||||||
|
"service_account_info": {
|
||||||
|
"description": "Contents of GSheets JSON credentials.",
|
||||||
|
"type": "string",
|
||||||
|
"x-encrypted-extra": True,
|
||||||
|
},
|
||||||
|
},
|
||||||
"type": "object",
|
"type": "object",
|
||||||
},
|
},
|
||||||
"preferred": False,
|
"preferred": False,
|
||||||
|
|
|
@ -35,7 +35,7 @@ def test_validate_parameters_simple(
|
||||||
)
|
)
|
||||||
|
|
||||||
parameters: GSheetsParametersType = {
|
parameters: GSheetsParametersType = {
|
||||||
"credentials_info": {},
|
"service_account_info": "",
|
||||||
"catalog": {},
|
"catalog": {},
|
||||||
}
|
}
|
||||||
errors = GSheetsEngineSpec.validate_parameters(parameters)
|
errors = GSheetsEngineSpec.validate_parameters(parameters)
|
||||||
|
@ -63,7 +63,7 @@ def test_validate_parameters_catalog(
|
||||||
]
|
]
|
||||||
|
|
||||||
parameters: GSheetsParametersType = {
|
parameters: GSheetsParametersType = {
|
||||||
"credentials_info": {},
|
"service_account_info": "",
|
||||||
"catalog": {
|
"catalog": {
|
||||||
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
|
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
|
||||||
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
|
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
|
||||||
|
@ -137,7 +137,7 @@ def test_validate_parameters_catalog_and_credentials(
|
||||||
]
|
]
|
||||||
|
|
||||||
parameters: GSheetsParametersType = {
|
parameters: GSheetsParametersType = {
|
||||||
"credentials_info": {},
|
"service_account_info": "",
|
||||||
"catalog": {
|
"catalog": {
|
||||||
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
|
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
|
||||||
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
|
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
|
||||||
|
|
Loading…
Reference in New Issue