feat(dbc ui): Adding Google Sheets Dynamic Form (#15801)

* feat: Make Google Sheets Dyanmic (#15576)

* first draft

* second draft

* added tests

* first draft

* added table_catalog

* remove console.log

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* revisions

* save this for now

* working form

* save disable on public sheets

* refactor somethings

* saving this for now

* working edit

* add back query to schema

* working add

* fix styling

* fixing x

* fix linting

* prettier

* fix some type issues

* more lint fixes

* remove unused dependency

* fix linint

* fix validation

* pylint bypass

* pylint bypass

* fix this

* fix mypy

* yerp

* fix test

* fix test

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* wrap add sheets

* fix linting issues

* fix unit test

* ignore typing

* fix editting and paste issues

* remove query

* fix this

* fix test

* add test back

* fix error messaging

* update url messaging on error

* change error type

* Update superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx

Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>

* add errors for sheets with no name

* fix

* fix messaging for gsheets

* stop pylint

* update line

Co-authored-by: AAfghahi <48933336+AAfghahi@users.noreply.github.com>
Co-authored-by: Arash <arash.afghahi@gmail.com>
Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>
This commit is contained in:
Hugh A. Miles II 2021-07-28 15:00:27 -04:00 committed by GitHub
parent 3f6c81b621
commit bfe7eb9a7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 464 additions and 94 deletions

View File

@ -217,7 +217,7 @@ max-nested-blocks=5
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=88
max-line-length=90
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$

View File

@ -19,19 +19,21 @@
import React, { FormEvent, useState } from 'react';
import { SupersetTheme, JsonObject, t } from '@superset-ui/core';
import { InputProps } from 'antd/lib/input';
import { Switch, Select, Button } from 'src/common/components';
import { Input, Switch, Select, Button } from 'src/common/components';
import InfoTooltip from 'src/components/InfoTooltip';
import ValidatedInput from 'src/components/Form/LabeledErrorBoundInput';
import FormLabel from 'src/components/Form/FormLabel';
import { DeleteFilled } from '@ant-design/icons';
import { DeleteFilled, CloseOutlined } from '@ant-design/icons';
import {
formScrollableStyles,
validatedFormStyles,
CredentialInfoForm,
toggleStyle,
infoTooltip,
StyledFooterButton,
StyledCatalogTable,
} from './styles';
import { DatabaseForm, DatabaseObject } from '../types';
import { CatalogObject, DatabaseForm, DatabaseObject } from '../types';
enum CredentialInfoOptions {
jsonUpload,
@ -46,6 +48,7 @@ export const FormFieldOrder = [
'password',
'database_name',
'credentials_info',
'catalog',
'query',
'encryption',
];
@ -58,7 +61,10 @@ interface FieldPropTypes {
onParametersUploadFileChange: (value: any) => string;
changeMethods: { onParametersChange: (value: any) => string } & {
onChange: (value: any) => string;
} & { onParametersUploadFileChange: (value: any) => string };
} & { onParametersUploadFileChange: (value: any) => string } & {
onAddTableCatalog: () => void;
onRemoveTableCatalog: (idx: number) => void;
};
validationErrors: JsonObject | null;
getValidation: () => void;
db?: DatabaseObject;
@ -187,6 +193,89 @@ const CredentialsInfo = ({
);
};
const TableCatalog = ({
required,
changeMethods,
getValidation,
validationErrors,
db,
}: FieldPropTypes) => {
const tableCatalog = db?.catalog || [];
const catalogError = validationErrors || {};
return (
<StyledCatalogTable>
<div className="catalog-type-select">
<FormLabel required>{t('Type of Google Sheets Allowed')}</FormLabel>
<Select style={{ width: '100%' }} defaultValue="true" disabled>
<Select.Option value="true" key={1}>
{t('Publicly shared sheets only')}
</Select.Option>
</Select>
</div>
<h4 className="gsheet-title">
{t('Connect Google Sheets as tables to this database')}
</h4>
<div>
{tableCatalog?.map((sheet: CatalogObject, idx: number) => (
<>
<FormLabel className="catalog-label" required>
{t('Google Sheet Name and URL')}
</FormLabel>
<div className="catalog-name">
<Input
className="catalog-name-input"
placeholder={t('Enter a name for this sheet')}
onChange={e => {
changeMethods.onParametersChange({
target: {
type: `catalog-${idx}`,
name: 'name',
value: e.target.value,
},
});
}}
value={sheet.name}
/>
{tableCatalog?.length > 1 && (
<CloseOutlined
className="catalog-delete"
onClick={() => changeMethods.onRemoveTableCatalog(idx)}
/>
)}
</div>
<ValidatedInput
className="catalog-name-url"
required={required}
validationMethods={{ onBlur: getValidation }}
errorMessage={catalogError[sheet.name]}
placeholder={t('Paste the shareable Google Sheet URL here')}
onChange={(e: { target: { value: any } }) =>
changeMethods.onParametersChange({
target: {
type: `catalog-${idx}`,
name: 'value',
value: e.target.value,
},
})
}
value={sheet.value}
/>
</>
))}
<StyledFooterButton
className="catalog-add-btn"
onClick={() => {
changeMethods.onAddTableCatalog();
}}
>
+ {t('Add sheet')}
</StyledFooterButton>
</div>
</StyledCatalogTable>
);
};
const hostField = ({
required,
changeMethods,
@ -300,18 +389,22 @@ const displayField = ({
validationErrors,
db,
}: FieldPropTypes) => (
<ValidatedInput
id="database_name"
name="database_name"
required
value={db?.database_name}
validationMethods={{ onBlur: getValidation }}
errorMessage={validationErrors?.database_name}
placeholder=""
label="Display Name"
onChange={changeMethods.onChange}
helpText={t('Pick a nickname for this database to display as in Superset.')}
/>
<>
<ValidatedInput
id="database_name"
name="database_name"
required
value={db?.database_name}
validationMethods={{ onBlur: getValidation }}
errorMessage={validationErrors?.database_name}
placeholder=""
label={t('Display Name')}
onChange={changeMethods.onChange}
helpText={t(
'Pick a nickname for this database to display as in Superset.',
)}
/>
</>
);
const queryField = ({
@ -375,6 +468,7 @@ const FORM_FIELD_MAP = {
query: queryField,
encryption: forceSSLField,
credentials_info: CredentialsInfo,
catalog: TableCatalog,
};
const DatabaseConnectionForm = ({
@ -382,6 +476,8 @@ const DatabaseConnectionForm = ({
onParametersChange,
onChange,
onParametersUploadFileChange,
onAddTableCatalog,
onRemoveTableCatalog,
validationErrors,
getValidation,
db,
@ -403,6 +499,8 @@ const DatabaseConnectionForm = ({
onParametersUploadFileChange?: (
event: FormEvent<InputProps> | { target: HTMLInputElement },
) => void;
onAddTableCatalog: () => void;
onRemoveTableCatalog: (idx: number) => void;
validationErrors: JsonObject | null;
getValidation: () => void;
}) => (
@ -426,6 +524,8 @@ const DatabaseConnectionForm = ({
onParametersChange,
onChange,
onParametersUploadFileChange,
onAddTableCatalog,
onRemoveTableCatalog,
},
validationErrors,
getValidation,

View File

@ -50,6 +50,7 @@ import {
DatabaseObject,
DatabaseForm,
CONFIGURATION_METHOD,
CatalogObject,
} from 'src/views/CRUD/data/database/types';
import Loading from 'src/components/Loading';
import ExtraOptions from './ExtraOptions';
@ -101,10 +102,15 @@ const errorAlertMapping = {
message: 'Invalid account information',
description: 'Either the username or password is incorrect.',
},
INVALID_PAYLOAD_SCHEMA: {
INVALID_PAYLOAD_SCHEMA_ERROR: {
message: 'Incorrect Fields',
description: 'Please make sure all fields are filled out correctly',
},
TABLE_DOES_NOT_EXIST_ERROR: {
message: 'URL could not be identified',
description:
'The URL could not be identified. Please check for typos and make sure that "Type of google sheet allowed" selection matches the input',
},
};
interface DatabaseModalProps {
addDangerToast: (msg: string) => void;
@ -126,6 +132,8 @@ enum ActionType {
textChange,
extraInputChange,
extraEditorChange,
addTableCatalogSheet,
removeTableCatalogSheet,
}
interface DBReducerPayloadType {
@ -161,7 +169,13 @@ type DBReducerActionType =
};
}
| {
type: ActionType.reset;
type: ActionType.reset | ActionType.addTableCatalogSheet;
}
| {
type: ActionType.removeTableCatalogSheet;
payload: {
indexToDelete: number;
};
}
| {
type: ActionType.configMethodChange;
@ -180,6 +194,9 @@ function dbReducer(
...(state || {}),
};
let query = '';
let deserializeExtraJSON = {};
let extra_json: DatabaseObject['extra_json'];
switch (action.type) {
case ActionType.extraEditorChange:
return {
@ -227,6 +244,29 @@ function dbReducer(
[action.payload.name]: action.payload.value,
};
case ActionType.parametersChange:
if (
trimmedState.catalog !== undefined &&
action.payload.type?.startsWith('catalog')
) {
// Formatting wrapping google sheets table catalog
const idx = action.payload.type?.split('-')[1];
const catalogToUpdate = trimmedState?.catalog[idx] || {};
catalogToUpdate[action.payload.name] = action.payload.value;
const paramatersCatalog = {};
// eslint-disable-next-line array-callback-return
trimmedState.catalog?.map((item: CatalogObject) => {
paramatersCatalog[item.name] = item.value;
});
return {
...trimmedState,
parameters: {
...trimmedState.parameters,
catalog: paramatersCatalog,
},
};
}
return {
...trimmedState,
parameters: {
@ -234,6 +274,22 @@ function dbReducer(
[action.payload.name]: action.payload.value,
},
};
case ActionType.addTableCatalogSheet:
if (trimmedState.catalog !== undefined) {
return {
...trimmedState,
catalog: [...trimmedState.catalog, { name: '', value: '' }],
};
}
return {
...trimmedState,
catalog: [{ name: '', value: '' }],
};
case ActionType.removeTableCatalogSheet:
trimmedState.catalog?.splice(action.payload.indexToDelete, 1);
return {
...trimmedState,
};
case ActionType.editorChange:
return {
...trimmedState,
@ -246,10 +302,8 @@ function dbReducer(
};
case ActionType.fetched:
// convert all the keys in this payload into strings
// eslint-disable-next-line no-case-declarations
let deserializeExtraJSON = {};
if (action.payload.extra) {
const extra_json = {
extra_json = {
...JSON.parse(action.payload.extra || ''),
} as DatabaseObject['extra_json'];
@ -262,13 +316,6 @@ function dbReducer(
};
}
if (action.payload?.parameters?.query) {
// convert query into URI params string
query = new URLSearchParams(
action.payload.parameters.query as string,
).toString();
}
if (
action.payload.backend === 'bigquery' &&
action.payload.configuration_method ===
@ -288,6 +335,46 @@ function dbReducer(
};
}
if (
action.payload.backend === 'gsheets' &&
action.payload.configuration_method ===
CONFIGURATION_METHOD.DYNAMIC_FORM &&
extra_json?.engine_params?.catalog !== undefined
) {
// pull catalog from engine params
const engineParamsCatalog = extra_json?.engine_params?.catalog;
return {
...action.payload,
engine: action.payload.backend,
configuration_method: action.payload.configuration_method,
extra_json: deserializeExtraJSON,
catalog: Object.keys(engineParamsCatalog).map(e => ({
name: e,
value: engineParamsCatalog[e],
})),
} as DatabaseObject;
}
if (action.payload?.parameters?.query) {
// convert query into URI params string
query = new URLSearchParams(
action.payload.parameters.query as string,
).toString();
return {
...action.payload,
encrypted_extra: action.payload.encrypted_extra || '',
engine: action.payload.backend || trimmedState.engine,
configuration_method: action.payload.configuration_method,
extra_json: deserializeExtraJSON,
parameters: {
...action.payload.parameters,
query,
},
};
}
return {
...action.payload,
encrypted_extra: action.payload.encrypted_extra || '',
@ -296,9 +383,9 @@ function dbReducer(
extra_json: deserializeExtraJSON,
parameters: {
...action.payload.parameters,
query,
},
};
case ActionType.dbSelected:
return {
...action.payload,
@ -319,7 +406,9 @@ const serializeExtra = (extraJson: DatabaseObject['extra_json']) =>
JSON.stringify({
...extraJson,
metadata_params: JSON.parse((extraJson?.metadata_params as string) || '{}'),
engine_params: JSON.parse((extraJson?.engine_params as string) || '{}'),
engine_params: JSON.parse(
((extraJson?.engine_params as unknown) as string) || '{}',
),
schemas_allowed_for_csv_upload:
(extraJson?.schemas_allowed_for_csv_upload as string) || '[]',
});
@ -369,7 +458,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
t('database'),
addDangerToast,
);
const isDynamic = (engine: string | undefined) =>
availableDbs?.databases.filter(
(DB: DatabaseObject) => DB.backend === engine || DB.engine === engine,
@ -435,7 +523,10 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
.replace(/&/g, '","')
.replace(/=/g, '":"')}"}`,
);
} else if (dbToUpdate?.parameters?.query === '') {
} else if (
dbToUpdate?.parameters?.query === '' &&
'query' in dbModel.parameters
) {
dbToUpdate.parameters.query = {};
}
@ -466,6 +557,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
}
}
if (dbToUpdate.parameters.catalog) {
// need to stringify gsheets catalog to allow it to be seralized
dbToUpdate.extra_json = {
engine_params: JSON.stringify({
catalog: dbToUpdate.parameters.catalog,
}),
};
}
if (dbToUpdate?.extra_json) {
// convert extra_json to back to string
dbToUpdate.extra = serializeExtra(dbToUpdate?.extra_json);
@ -545,6 +645,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
engine,
},
});
setDB({ type: ActionType.addTableCatalogSheet });
};
const renderAvailableSelector = () => (
@ -816,6 +917,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
value: target.value,
})
}
onAddTableCatalog={() =>
setDB({ type: ActionType.addTableCatalogSheet })
}
onRemoveTableCatalog={(idx: number) =>
setDB({
type: ActionType.removeTableCatalogSheet,
payload: { indexToDelete: idx },
})
}
getValidation={() => getValidation(db)}
validationErrors={validationErrors}
/>
@ -928,6 +1038,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
value: target.value,
})
}
onAddTableCatalog={() =>
setDB({ type: ActionType.addTableCatalogSheet })
}
onRemoveTableCatalog={(idx: number) =>
setDB({
type: ActionType.removeTableCatalogSheet,
payload: { indexToDelete: idx },
})
}
getValidation={() => getValidation(db)}
validationErrors={validationErrors}
/>
@ -1030,7 +1149,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
</>
) : (
<>
{/* Step 1 */}
{/* Dyanmic Form Step 1 */}
{!isLoading &&
(!db ? (
<SelectDatabaseStyles>
@ -1073,6 +1192,15 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
db={db}
sslForced={sslForced}
dbModel={dbModel}
onAddTableCatalog={() => {
setDB({ type: ActionType.addTableCatalogSheet });
}}
onRemoveTableCatalog={(idx: number) => {
setDB({
type: ActionType.removeTableCatalogSheet,
payload: { indexToDelete: idx },
});
}}
onParametersChange={({
target,
}: {

View File

@ -537,3 +537,43 @@ export const StyledStickyHeader = styled.div`
z-index: ${({ theme }) => theme.zIndex.max};
background: ${({ theme }) => theme.colors.grayscale.light5};
`;
export const StyledCatalogTable = styled.div`
margin-bottom: 16px;
.catalog-type-select {
margin: 0 0 40px;
}
.gsheet-title {
font-size: ${({ theme }) => theme.typography.sizes.l * 1.1}px;
font-weight: bold;
margin: ${({ theme }) => theme.gridUnit * 6}px 0 16px;
}
.catalog-label {
margin: 0 0 8px;
}
.catalog-name {
display: flex;
.catalog-name-input {
width: 95%;
}
}
.catalog-name-url {
margin: 4px 0;
width: 95%;
}
.catalog-delete {
align-self: center;
background: ${({ theme }) => theme.colors.grayscale.light4};
margin: 5px;
}
.catalog-add-btn {
width: 95%;
}
`;

View File

@ -21,6 +21,11 @@ type DatabaseUser = {
last_name: string;
};
export type CatalogObject = {
name: string;
value: string;
};
export type DatabaseObject = {
// Connection + general
id?: number;
@ -41,10 +46,14 @@ export type DatabaseObject = {
encryption?: boolean;
credentials_info?: string;
query?: string | object;
catalog?: {};
};
configuration_method: CONFIGURATION_METHOD;
engine?: string;
// Gsheets temporary storage
catalog?: Array<CatalogObject>;
// Performance
cache_timeout?: string;
allow_run_async?: boolean;
@ -65,7 +74,9 @@ export type DatabaseObject = {
// Extra
extra_json?: {
engine_params?: {} | string;
engine_params?: {
catalog: Record<any, any> | string;
};
metadata_params?: {} | string;
metadata_cache_timeout?: {
schema_cache_timeout?: number; // in Performance

View File

@ -674,7 +674,11 @@ export function useDatabaseValidation() {
message,
}: {
error_type: string;
extra: { invalid?: string[]; missing?: string[] };
extra: {
invalid?: string[];
missing?: string[];
name: string;
};
message: string;
},
) => {
@ -682,6 +686,13 @@ export function useDatabaseValidation() {
// error can't be mapped to a parameter
// so leave it alone
if (extra.invalid) {
if (extra.invalid[0] === 'catalog') {
return {
...obj,
[extra.name]: message,
error_type,
};
}
return {
...obj,
[extra.invalid[0]]: message,

View File

@ -64,7 +64,7 @@ class ValidateDatabaseParametersCommand(BaseCommand):
),
)
engine_spec = engine_specs[engine]
if not issubclass(engine_spec, BasicParametersMixin):
if not hasattr(engine_spec, "parameters_schema"):
raise InvalidEngineError(
SupersetError(
message=__(
@ -85,7 +85,9 @@ class ValidateDatabaseParametersCommand(BaseCommand):
)
# perform initial validation
errors = engine_spec.validate_parameters(self._properties.get("parameters", {}))
errors = engine_spec.validate_parameters( # type: ignore
self._properties.get("parameters", {})
)
if errors:
raise InvalidParametersError(errors)
@ -96,9 +98,8 @@ class ValidateDatabaseParametersCommand(BaseCommand):
encrypted_extra = {}
# try to connect
sqlalchemy_uri = engine_spec.build_sqlalchemy_uri(
self._properties.get("parameters"), # type: ignore
encrypted_extra,
sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( # type: ignore
self._properties.get("parameters"), encrypted_extra,
)
if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri():
sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted

View File

@ -19,13 +19,18 @@ import re
from contextlib import closing
from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
from apispec import APISpec
from apispec.ext.marshmallow import MarshmallowPlugin
from flask import g
from flask_babel import gettext as __
from marshmallow import fields, Schema
from marshmallow.exceptions import ValidationError
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import URL
from typing_extensions import TypedDict
from superset import security_manager
from superset.databases.schemas import encrypted_field_properties
from superset.db_engine_specs.sqlite import SqliteEngineSpec
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
@ -35,11 +40,16 @@ if TYPE_CHECKING:
SYNTAX_ERROR_REGEX = re.compile('SQLError: near "(?P<server_error>.*?)": syntax error')
ma_plugin = MarshmallowPlugin()
class GSheetsParametersSchema(Schema):
catalog = fields.Dict()
class GSheetsParametersType(TypedDict):
credentials_info: Dict[str, Any]
query: Dict[str, Any]
table_catalog: Dict[str, str]
catalog: Dict[str, str]
class GSheetsEngineSpec(SqliteEngineSpec):
@ -50,6 +60,10 @@ class GSheetsEngineSpec(SqliteEngineSpec):
allows_joins = True
allows_subqueries = True
parameters_schema = GSheetsParametersSchema()
default_driver = "apsw"
sqlalchemy_uri_placeholder = "gsheets://"
custom_errors: Dict[Pattern[str], Tuple[str, SupersetErrorType, Dict[str, Any]]] = {
SYNTAX_ERROR_REGEX: (
__(
@ -87,16 +101,64 @@ class GSheetsEngineSpec(SqliteEngineSpec):
return {"metadata": metadata["extra"]}
@classmethod
def build_sqlalchemy_uri(
cls,
_: GSheetsParametersType,
encrypted_extra: Optional[ # pylint: disable=unused-argument
Dict[str, Any]
] = None,
) -> str: # pylint: disable=unused-variable
return "gsheets://"
@classmethod
def get_parameters_from_uri(
cls, encrypted_extra: Optional[Dict[str, str]] = None,
) -> Any:
# Building parameters from encrypted_extra and uri
if encrypted_extra:
return {**encrypted_extra}
raise ValidationError("Invalid service credentials")
@classmethod
def parameters_json_schema(cls) -> Any:
"""
Return configuration parameters as OpenAPI.
"""
if not cls.parameters_schema:
return None
spec = APISpec(
title="Database Parameters",
version="1.0.0",
openapi_version="3.0.0",
plugins=[ma_plugin],
)
ma_plugin.init_spec(spec)
ma_plugin.converter.add_attribute_function(encrypted_field_properties)
spec.components.schema(cls.__name__, schema=cls.parameters_schema)
return spec.to_dict()["components"]["schemas"][cls.__name__]
@classmethod
def validate_parameters(
cls, parameters: GSheetsParametersType,
) -> List[SupersetError]:
errors: List[SupersetError] = []
credentials_info = parameters.get("credentials_info")
table_catalog = parameters.get("table_catalog", {})
table_catalog = parameters.get("catalog", {})
if not table_catalog:
errors.append(
SupersetError(
message="URL is required",
error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR,
level=ErrorLevel.WARNING,
extra={"invalid": ["catalog"], "name": "", "url": ""},
),
)
return errors
# We need a subject in case domain wide delegation is set, otherwise the
@ -110,17 +172,27 @@ class GSheetsEngineSpec(SqliteEngineSpec):
)
conn = engine.connect()
for name, url in table_catalog.items():
if not name:
errors.append(
SupersetError(
message="Sheet name is required",
error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR,
level=ErrorLevel.WARNING,
extra={"invalid": [], "name": name, "url": url},
),
)
try:
results = conn.execute(f'SELECT * FROM "{url}" LIMIT 1')
results.fetchall()
except Exception: # pylint: disable=broad-except
errors.append(
SupersetError(
message=f"Unable to connect to spreadsheet {name} at {url}",
message="URL could not be identified",
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
level=ErrorLevel.WARNING,
extra={"name": name, "url": url},
extra={"invalid": ["catalog"], "name": name, "url": url},
),
)
return errors

View File

@ -39,6 +39,7 @@ from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.db_engine_specs.postgres import PostgresEngineSpec
from superset.db_engine_specs.redshift import RedshiftEngineSpec
from superset.db_engine_specs.bigquery import BigQueryEngineSpec
from superset.db_engine_specs.gsheets import GSheetsEngineSpec
from superset.db_engine_specs.hana import HanaEngineSpec
from superset.errors import SupersetError
from superset.models.core import Database, ConfigurationMethod
@ -1438,6 +1439,7 @@ class TestDatabaseApi(SupersetTestCase):
PostgresEngineSpec: {"psycopg2"},
BigQueryEngineSpec: {"bigquery"},
MySQLEngineSpec: {"mysqlconnector", "mysqldb"},
GSheetsEngineSpec: {"apsw"},
RedshiftEngineSpec: {"psycopg2"},
HanaEngineSpec: {""},
}
@ -1565,6 +1567,18 @@ class TestDatabaseApi(SupersetTestCase):
"preferred": False,
"sqlalchemy_uri_placeholder": "redshift+psycopg2://user:password@host:port/dbname[?key=value&key=value...]",
},
{
"available_drivers": ["apsw"],
"default_driver": "apsw",
"engine": "gsheets",
"name": "Google Sheets",
"parameters": {
"properties": {"catalog": {"type": "object"},},
"type": "object",
},
"preferred": False,
"sqlalchemy_uri_placeholder": "gsheets://",
},
{
"available_drivers": ["mysqlconnector", "mysqldb"],
"default_driver": "mysqldb",

View File

@ -37,11 +37,27 @@ def test_validate_parameters_simple(
parameters: GSheetsParametersType = {
"credentials_info": {},
"query": {},
"table_catalog": {},
"catalog": {},
}
errors = GSheetsEngineSpec.validate_parameters(parameters)
assert errors == []
assert errors == [
SupersetError(
message="URL is required",
error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR,
level=ErrorLevel.WARNING,
extra={
"invalid": ["catalog"],
"name": "",
"url": "",
"issue_codes": [
{
"code": 1018,
"message": "Issue 1018 - One or more parameters needed to configure a database are missing.",
}
],
},
)
]
def test_validate_parameters_catalog(
@ -66,72 +82,57 @@ def test_validate_parameters_catalog(
parameters: GSheetsParametersType = {
"credentials_info": {},
"query": {},
"table_catalog": {
"catalog": {
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
"not_a_sheet": "https://www.google.com/",
},
}
errors = GSheetsEngineSpec.validate_parameters(parameters)
errors = GSheetsEngineSpec.validate_parameters(parameters) # ignore: type
assert errors == [
SupersetError(
message=(
"Unable to connect to spreadsheet private_sheet at "
"https://docs.google.com/spreadsheets/d/1/edit"
),
message="URL could not be identified",
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
level=ErrorLevel.WARNING,
extra={
"invalid": ["catalog"],
"name": "private_sheet",
"url": "https://docs.google.com/spreadsheets/d/1/edit",
"issue_codes": [
{
"code": 1003,
"message": (
"Issue 1003 - There is a syntax error in the SQL query. "
"Perhaps there was a misspelling or a typo."
),
"message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.",
},
{
"code": 1005,
"message": (
"Issue 1005 - The table was deleted or renamed in the "
"database."
),
"message": "Issue 1005 - The table was deleted or renamed in the database.",
},
],
},
),
SupersetError(
message=(
"Unable to connect to spreadsheet not_a_sheet at "
"https://www.google.com/"
),
message="URL could not be identified",
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
level=ErrorLevel.WARNING,
extra={
"invalid": ["catalog"],
"name": "not_a_sheet",
"url": "https://www.google.com/",
"issue_codes": [
{
"code": 1003,
"message": (
"Issue 1003 - There is a syntax error in the SQL query. "
"Perhaps there was a misspelling or a typo."
),
"message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.",
},
{
"code": 1005,
"message": (
"Issue 1005 - The table was deleted or renamed in the "
"database.",
),
"message": "Issue 1005 - The table was deleted or renamed in the database.",
},
],
},
),
]
create_engine.assert_called_with(
"gsheets://", service_account_info={}, subject="admin@example.com",
)
@ -159,44 +160,36 @@ def test_validate_parameters_catalog_and_credentials(
parameters: GSheetsParametersType = {
"credentials_info": {},
"query": {},
"table_catalog": {
"catalog": {
"private_sheet": "https://docs.google.com/spreadsheets/d/1/edit",
"public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1",
"not_a_sheet": "https://www.google.com/",
},
}
errors = GSheetsEngineSpec.validate_parameters(parameters)
errors = GSheetsEngineSpec.validate_parameters(parameters) # ignore: type
assert errors == [
SupersetError(
message=(
"Unable to connect to spreadsheet not_a_sheet at "
"https://www.google.com/"
),
message="URL could not be identified",
error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
level=ErrorLevel.WARNING,
extra={
"invalid": ["catalog"],
"name": "not_a_sheet",
"url": "https://www.google.com/",
"issue_codes": [
{
"code": 1003,
"message": (
"Issue 1003 - There is a syntax error in the SQL query. "
"Perhaps there was a misspelling or a typo."
),
"message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.",
},
{
"code": 1005,
"message": (
"Issue 1005 - The table was deleted or renamed in the "
"database.",
),
"message": "Issue 1005 - The table was deleted or renamed in the database.",
},
],
},
),
)
]
create_engine.assert_called_with(
"gsheets://", service_account_info={}, subject="admin@example.com",
)