From 01aede0652f15e892a21c3b0c0d88db16e60b1dc Mon Sep 17 00:00:00 2001 From: Will Barrett Date: Wed, 17 Jun 2020 13:42:13 -0700 Subject: [PATCH] refactor: Break up superset/views/core.py (#10078) * Remove unreferenced function from views/core.py * Remove excess constants from views/core.py * Extract CssTemplate-related views to their own file from core.py * Remove duplicate constant declaration and make the constant less racist * Move free-floating functions in views/core.py to views/utils.py * Move AccessRequestsModelView out of views/core.py into its own module * Move health checks and KV ModelView out of core.py and into their own modules * Move R model view to its own module * Move after-request header setting to views/base.py from views/core.py * black * mypy * isort * Fix reference to imported app * pylint * Fix some imports * Add some missing view imports * Fix a missing import --- .pylintrc | 2 +- superset/app.py | 14 +- superset/views/__init__.py | 4 + superset/views/access_requests.py | 46 +++++ superset/views/base.py | 17 +- superset/views/core.py | 321 ++---------------------------- superset/views/css_templates.py | 46 +++++ superset/views/health.py | 36 ++++ superset/views/key_value.py | 56 ++++++ superset/views/redirects.py | 60 ++++++ superset/views/utils.py | 155 ++++++++++++++- tests/core_tests.py | 5 +- 12 files changed, 440 insertions(+), 322 deletions(-) create mode 100644 superset/views/access_requests.py create mode 100644 superset/views/css_templates.py create mode 100644 superset/views/health.py create mode 100644 superset/views/key_value.py create mode 100644 superset/views/redirects.py diff --git a/.pylintrc b/.pylintrc index 70ad8bc5b1..42b8783e7d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -48,7 +48,7 @@ unsafe-load-any-extension=no # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code -extension-pkg-whitelist= +extension-pkg-whitelist=pyarrow # Allow optimization of some AST trees. This will activate a peephole AST # optimizer, which will apply various small optimizations. For instance, it can diff --git a/superset/app.py b/superset/app.py index b36df75a1d..3e83fd581a 100644 --- a/superset/app.py +++ b/superset/app.py @@ -145,14 +145,12 @@ class SupersetAppInitializer: AnnotationModelView, ) from superset.views.api import Api - from superset.views.core import ( - AccessRequestsModelView, - KV, - R, - Superset, - CssTemplateModelView, - CssTemplateAsyncModelView, - ) + from superset.views.core import Superset + from superset.views.redirects import R + from superset.views.key_value import KV + from superset.views.access_requests import AccessRequestsModelView + from superset.views.css_templates import CssTemplateAsyncModelView + from superset.views.css_templates import CssTemplateModelView from superset.charts.api import ChartRestApi from superset.views.chart.views import SliceModelView, SliceAsync from superset.dashboards.api import DashboardRestApi diff --git a/superset/views/__init__.py b/superset/views/__init__.py index b59244fa1c..9575ed1d53 100644 --- a/superset/views/__init__.py +++ b/superset/views/__init__.py @@ -15,12 +15,16 @@ # specific language governing permissions and limitations # under the License. from . import ( + access_requests, annotations, api, base, core, + css_templates, dashboard, datasource, + health, + redirects, schedules, sql_lab, tags, diff --git a/superset/views/access_requests.py b/superset/views/access_requests.py new file mode 100644 index 0000000000..abcf4be501 --- /dev/null +++ b/superset/views/access_requests.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from flask_appbuilder.models.sqla.interface import SQLAInterface +from flask_babel import lazy_gettext as _ + +from superset.constants import RouteMethod +from superset.views.base import DeleteMixin, SupersetModelView +from superset.views.core import DAR + + +class AccessRequestsModelView( # pylint: disable=too-many-ancestors + SupersetModelView, DeleteMixin +): + datamodel = SQLAInterface(DAR) + include_route_methods = RouteMethod.CRUD_SET + list_columns = [ + "username", + "user_roles", + "datasource_link", + "roles_with_datasource", + "created_on", + ] + order_columns = ["created_on"] + base_order = ("changed_on", "desc") + label_columns = { + "username": _("User"), + "user_roles": _("User Roles"), + "database": _("Database URL"), + "datasource_link": _("Datasource"), + "roles_with_datasource": _("Roles to grant"), + "created_on": _("Created On"), + } diff --git a/superset/views/base.py b/superset/views/base.py index e9dab544f1..b95be11dfa 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -375,7 +375,7 @@ class YamlExportMixin: # pylint: disable=too-few-public-methods class DeleteMixin: # pylint: disable=too-few-public-methods - def _delete(self: BaseView, primary_key: int,) -> None: + def _delete(self: BaseView, primary_key: int) -> None: """ Delete function logic, override to implement diferent logic deletes the record with primary_key = primary_key @@ -520,3 +520,18 @@ def bind_field( FlaskForm.Meta.bind_field = bind_field + + +@superset_app.after_request +def apply_http_headers(response: Response) -> Response: + """Applies the configuration's http headers to all responses""" + + # HTTP_HEADERS is deprecated, this provides backwards compatibility + response.headers.extend( # type: ignore + {**config["OVERRIDE_HTTP_HEADERS"], **config["HTTP_HEADERS"]} + ) + + for k, v in config["DEFAULT_HTTP_HEADERS"].items(): + if k not in response.headers: + response.headers[k] = v + return response diff --git a/superset/views/core.py b/superset/views/core.py index ad0343cdc2..e08d92a189 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -20,22 +20,19 @@ import re from collections import defaultdict from contextlib import closing from datetime import datetime, timedelta -from typing import Any, Callable, cast, Dict, List, Optional, Union +from typing import Any, cast, Dict, List, Optional, Union from urllib import parse import backoff -import msgpack import pandas as pd -import pyarrow as pa import simplejson as json from flask import abort, flash, g, Markup, redirect, render_template, request, Response from flask_appbuilder import expose from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_appbuilder.security.decorators import has_access, has_access_api from flask_appbuilder.security.sqla import models as ab_models -from flask_appbuilder.security.sqla.models import User from flask_babel import gettext as __, lazy_gettext as _ -from sqlalchemy import and_, Integer, or_, select +from sqlalchemy import and_, or_, select from sqlalchemy.engine.url import make_url from sqlalchemy.exc import ( ArgumentError, @@ -51,17 +48,14 @@ from superset import ( app, appbuilder, conf, - dataframe, db, event_logger, get_feature_flags, is_feature_enabled, - result_set, results_backend, results_backend_use_msgpack, security_manager, sql_lab, - talisman, viz, ) from superset.connectors.connector_registry import ConnectorRegistry @@ -71,18 +65,13 @@ from superset.connectors.sqla.models import ( SqlMetric, TableColumn, ) -from superset.constants import RouteMethod -from superset.errors import ErrorLevel, SupersetError, SupersetErrorType from superset.exceptions import ( CertificateException, DatabaseNotFound, - QueryObjectValidationError, SupersetException, - SupersetSecurityException, SupersetTimeoutException, ) from superset.jinja_context import get_template_processor -from superset.models.core import Database from superset.models.dashboard import Dashboard from superset.models.datasource_access_request import DatasourceAccessRequest from superset.models.slice import Slice @@ -98,12 +87,8 @@ from superset.typing import FlaskResponse from superset.utils import core as utils, dashboard_import_export from superset.utils.dashboard_filter_scopes_converter import copy_filter_scopes from superset.utils.dates import now_as_float -from superset.utils.decorators import etag_cache, stats_timing -from superset.views.database.filters import DatabaseFilter -from superset.views.utils import get_dashboard_extra_filters -from superset.viz import BaseViz - -from .base import ( +from superset.utils.decorators import etag_cache +from superset.views.base import ( api, BaseSupersetView, check_ownership, @@ -111,7 +96,6 @@ from .base import ( create_table_permissions, CsvResponse, data_payload_response, - DeleteMixin, generate_download_headers, get_error_msg, get_user_roles, @@ -119,16 +103,23 @@ from .base import ( json_error_response, json_errors_response, json_success, - SupersetModelView, validate_sqlatable, ) -from .utils import ( +from superset.views.database.filters import DatabaseFilter +from superset.views.utils import ( + _deserialize_results_payload, apply_display_max_row_limit, bootstrap_user_data, + check_datasource_perms, + check_slice_perms, + get_cta_schema_name, + get_dashboard_extra_filters, get_datasource_info, get_form_data, get_viz, + is_owner, ) +from superset.viz import BaseViz config = app.config CACHE_DEFAULT_TIMEOUT = config["CACHE_DEFAULT_TIMEOUT"] @@ -153,249 +144,9 @@ DATABASE_KEYS = [ ] -ALL_DATASOURCE_ACCESS_ERR = __( - "This endpoint requires the `all_datasource_access` permission" -) DATASOURCE_MISSING_ERR = __("The data source seems to have been deleted") -ACCESS_REQUEST_MISSING_ERR = __("The access requests seem to have been deleted") USER_MISSING_ERR = __("The user seems to have been deleted") -FORM_DATA_KEY_BLACKLIST: List[str] = [] -if not config["ENABLE_JAVASCRIPT_CONTROLS"]: - FORM_DATA_KEY_BLACKLIST = ["js_tooltip", "js_onclick_href", "js_data_mutator"] - - -def get_database_access_error_msg(database_name: str) -> str: - return __( - "This view requires the database %(name)s or " - "`all_datasource_access` permission", - name=database_name, - ) - - -def is_owner(obj: Union[Dashboard, Slice], user: User) -> bool: - """ Check if user is owner of the slice """ - return obj and user in obj.owners - - -def check_datasource_perms( - self: "Superset", - datasource_type: Optional[str] = None, - datasource_id: Optional[int] = None, -) -> None: - """ - Check if user can access a cached response from explore_json. - - This function takes `self` since it must have the same signature as the - the decorated method. - - :param datasource_type: The datasource type, i.e., 'druid' or 'table' - :param datasource_id: The datasource ID - :raises SupersetSecurityException: If the user cannot access the resource - """ - - form_data = get_form_data()[0] - - try: - datasource_id, datasource_type = get_datasource_info( - datasource_id, datasource_type, form_data - ) - except SupersetException as ex: - raise SupersetSecurityException( - SupersetError( - error_type=SupersetErrorType.FAILED_FETCHING_DATASOURCE_INFO_ERROR, - level=ErrorLevel.ERROR, - message=str(ex), - ) - ) - - if datasource_type is None: - raise SupersetSecurityException( - SupersetError( - error_type=SupersetErrorType.UNKNOWN_DATASOURCE_TYPE_ERROR, - level=ErrorLevel.ERROR, - message="Could not determine datasource type", - ) - ) - - viz_obj = get_viz( - datasource_type=datasource_type, - datasource_id=datasource_id, - form_data=form_data, - force=False, - ) - - security_manager.assert_viz_permission(viz_obj) - - -def check_slice_perms(self: "Superset", slice_id: int) -> None: - """ - Check if user can access a cached response from slice_json. - - This function takes `self` since it must have the same signature as the - the decorated method. - """ - - form_data, slc = get_form_data(slice_id, use_slice_data=True) - - if slc: - viz_obj = get_viz( - datasource_type=slc.datasource.type, - datasource_id=slc.datasource.id, - form_data=form_data, - force=False, - ) - - security_manager.assert_viz_permission(viz_obj) - - -def _deserialize_results_payload( - payload: Union[bytes, str], query: Query, use_msgpack: Optional[bool] = False -) -> Dict[str, Any]: - logger.debug(f"Deserializing from msgpack: {use_msgpack}") - if use_msgpack: - with stats_timing( - "sqllab.query.results_backend_msgpack_deserialize", stats_logger - ): - ds_payload = msgpack.loads(payload, raw=False) - - with stats_timing("sqllab.query.results_backend_pa_deserialize", stats_logger): - pa_table = pa.deserialize(ds_payload["data"]) - - df = result_set.SupersetResultSet.convert_table_to_df(pa_table) - ds_payload["data"] = dataframe.df_to_records(df) or [] - - db_engine_spec = query.database.db_engine_spec - all_columns, data, expanded_columns = db_engine_spec.expand_data( - ds_payload["selected_columns"], ds_payload["data"] - ) - ds_payload.update( - {"data": data, "columns": all_columns, "expanded_columns": expanded_columns} - ) - - return ds_payload - else: - with stats_timing( - "sqllab.query.results_backend_json_deserialize", stats_logger - ): - return json.loads(payload) - - -def get_cta_schema_name( - database: Database, user: ab_models.User, schema: str, sql: str -) -> Optional[str]: - func: Optional[Callable[[Database, ab_models.User, str, str], str]] = config[ - "SQLLAB_CTAS_SCHEMA_NAME_FUNC" - ] - if not func: - return None - return func(database, user, schema, sql) - - -class AccessRequestsModelView(SupersetModelView, DeleteMixin): - datamodel = SQLAInterface(DAR) - include_route_methods = RouteMethod.CRUD_SET - list_columns = [ - "username", - "user_roles", - "datasource_link", - "roles_with_datasource", - "created_on", - ] - order_columns = ["created_on"] - base_order = ("changed_on", "desc") - label_columns = { - "username": _("User"), - "user_roles": _("User Roles"), - "database": _("Database URL"), - "datasource_link": _("Datasource"), - "roles_with_datasource": _("Roles to grant"), - "created_on": _("Created On"), - } - - -@talisman(force_https=False) -@app.route("/health") -def health() -> FlaskResponse: - return "OK" - - -@talisman(force_https=False) -@app.route("/healthcheck") -def healthcheck() -> FlaskResponse: - return "OK" - - -@talisman(force_https=False) -@app.route("/ping") -def ping() -> FlaskResponse: - return "OK" - - -class KV(BaseSupersetView): - - """Used for storing and retrieving key value pairs""" - - @event_logger.log_this - @has_access_api - @expose("/store/", methods=["POST"]) - def store(self) -> FlaskResponse: - try: - value = request.form.get("data") - obj = models.KeyValue(value=value) - db.session.add(obj) - db.session.commit() - except Exception as ex: - return json_error_response(utils.error_msg_from_exception(ex)) - return Response(json.dumps({"id": obj.id}), status=200) - - @event_logger.log_this - @has_access_api - @expose("//", methods=["GET"]) - def get_value(self, key_id: int) -> FlaskResponse: - try: - kv = db.session.query(models.KeyValue).filter_by(id=key_id).scalar() - if not kv: - return Response(status=404, content_type="text/plain") - except Exception as ex: - return json_error_response(utils.error_msg_from_exception(ex)) - return Response(kv.value, status=200, content_type="text/plain") - - -class R(BaseSupersetView): - - """used for short urls""" - - @event_logger.log_this - @expose("/") - def index(self, url_id: int) -> FlaskResponse: - url = db.session.query(models.Url).get(url_id) - if url and url.url: - explore_url = "//superset/explore/?" - if url.url.startswith(explore_url): - explore_url += f"r={url_id}" - return redirect(explore_url[1:]) - else: - return redirect(url.url[1:]) - else: - flash("URL to nowhere...", "danger") - return redirect("/") - - @event_logger.log_this - @has_access_api - @expose("/shortner/", methods=["POST"]) - def shortner(self) -> FlaskResponse: - url = request.form.get("data") - obj = models.Url(url=url) - db.session.add(obj) - db.session.commit() - return Response( - "{scheme}://{request.headers[Host]}/r/{obj.id}".format( - scheme=request.scheme, request=request, obj=obj - ), - mimetype="text/plain", - ) - class Superset(BaseSupersetView): """The base views for Superset!""" @@ -556,8 +307,9 @@ class Superset(BaseSupersetView): ) if not requests: - flash(ACCESS_REQUEST_MISSING_ERR, "alert") - return json_error_response(ACCESS_REQUEST_MISSING_ERR) + err = __("The access requests seem to have been deleted") + flash(err, "alert") + return json_error_response(err) # check if you can approve if security_manager.can_access_all_datasources() or check_ownership( @@ -2113,10 +1865,12 @@ class Superset(BaseSupersetView): def extra_table_metadata( self, database_id: int, table_name: str, schema: str ) -> FlaskResponse: - schema = utils.parse_js_uri_path_item(schema, eval_undefined=True) # type: ignore + parsed_schema = utils.parse_js_uri_path_item(schema, eval_undefined=True) table_name = utils.parse_js_uri_path_item(table_name) # type: ignore mydb = db.session.query(models.Database).filter_by(id=database_id).one() - payload = mydb.db_engine_spec.extra_table_metadata(mydb, table_name, schema) + payload = mydb.db_engine_spec.extra_table_metadata( + mydb, table_name, parsed_schema + ) return json_success(json.dumps(payload)) @has_access @@ -2913,38 +2667,3 @@ class Superset(BaseSupersetView): "Failed to fetch schemas allowed for csv upload in this database! " "Please contact your Superset Admin!" ) - - -class CssTemplateModelView(SupersetModelView, DeleteMixin): - datamodel = SQLAInterface(models.CssTemplate) - include_route_methods = RouteMethod.CRUD_SET - - list_title = _("CSS Templates") - show_title = _("Show CSS Template") - add_title = _("Add CSS Template") - edit_title = _("Edit CSS Template") - - list_columns = ["template_name"] - edit_columns = ["template_name", "css"] - add_columns = edit_columns - label_columns = {"template_name": _("Template Name")} - - -class CssTemplateAsyncModelView(CssTemplateModelView): - include_route_methods = {RouteMethod.API_READ} - list_columns = ["template_name", "css"] - - -@app.after_request -def apply_http_headers(response: Response) -> Response: - """Applies the configuration's http headers to all responses""" - - # HTTP_HEADERS is deprecated, this provides backwards compatibility - response.headers.extend( # type: ignore - {**config["OVERRIDE_HTTP_HEADERS"], **config["HTTP_HEADERS"]} - ) - - for k, v in config["DEFAULT_HTTP_HEADERS"].items(): - if k not in response.headers: - response.headers[k] = v - return response diff --git a/superset/views/css_templates.py b/superset/views/css_templates.py new file mode 100644 index 0000000000..0dff43c17a --- /dev/null +++ b/superset/views/css_templates.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from flask_appbuilder.models.sqla.interface import SQLAInterface +from flask_babel import lazy_gettext as _ + +from superset.constants import RouteMethod +from superset.models import core as models +from superset.views.base import DeleteMixin, SupersetModelView + + +class CssTemplateModelView( # pylint: disable=too-many-ancestors + SupersetModelView, DeleteMixin +): + datamodel = SQLAInterface(models.CssTemplate) + include_route_methods = RouteMethod.CRUD_SET + + list_title = _("CSS Templates") + show_title = _("Show CSS Template") + add_title = _("Add CSS Template") + edit_title = _("Edit CSS Template") + + list_columns = ["template_name"] + edit_columns = ["template_name", "css"] + add_columns = edit_columns + label_columns = {"template_name": _("Template Name")} + + +class CssTemplateAsyncModelView( # pylint: disable=too-many-ancestors + CssTemplateModelView +): + include_route_methods = {RouteMethod.API_READ} + list_columns = ["template_name", "css"] diff --git a/superset/views/health.py b/superset/views/health.py new file mode 100644 index 0000000000..876e7a5e13 --- /dev/null +++ b/superset/views/health.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from superset import app, talisman +from superset.typing import FlaskResponse + + +@talisman(force_https=False) +@app.route("/ping") +def ping() -> FlaskResponse: + return "OK" + + +@talisman(force_https=False) +@app.route("/healthcheck") +def healthcheck() -> FlaskResponse: + return "OK" + + +@talisman(force_https=False) +@app.route("/health") +def health() -> FlaskResponse: + return "OK" diff --git a/superset/views/key_value.py b/superset/views/key_value.py new file mode 100644 index 0000000000..117d664893 --- /dev/null +++ b/superset/views/key_value.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import simplejson as json +from flask import request, Response +from flask_appbuilder import expose +from flask_appbuilder.security.decorators import has_access_api + +from superset import db, event_logger +from superset.models import core as models +from superset.typing import FlaskResponse +from superset.utils import core as utils +from superset.views.base import BaseSupersetView, json_error_response + + +class KV(BaseSupersetView): + + """Used for storing and retrieving key value pairs""" + + @event_logger.log_this + @has_access_api + @expose("/store/", methods=["POST"]) + def store(self) -> FlaskResponse: # pylint: disable=no-self-use + try: + value = request.form.get("data") + obj = models.KeyValue(value=value) + db.session.add(obj) + db.session.commit() + except Exception as ex: # pylint: disable=broad-except + return json_error_response(utils.error_msg_from_exception(ex)) + return Response(json.dumps({"id": obj.id}), status=200) + + @event_logger.log_this + @has_access_api + @expose("//", methods=["GET"]) + def get_value(self, key_id: int) -> FlaskResponse: # pylint: disable=no-self-use + try: + kv = db.session.query(models.KeyValue).filter_by(id=key_id).scalar() + if not kv: + return Response(status=404, content_type="text/plain") + except Exception as ex: # pylint: disable=broad-except + return json_error_response(utils.error_msg_from_exception(ex)) + return Response(kv.value, status=200, content_type="text/plain") diff --git a/superset/views/redirects.py b/superset/views/redirects.py new file mode 100644 index 0000000000..02dc587e71 --- /dev/null +++ b/superset/views/redirects.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from flask import flash, request, Response +from flask_appbuilder import expose +from flask_appbuilder.security.decorators import has_access_api +from werkzeug.utils import redirect + +from superset import db, event_logger +from superset.models import core as models +from superset.typing import FlaskResponse +from superset.views.base import BaseSupersetView + + +class R(BaseSupersetView): # pylint: disable=invalid-name + + """used for short urls""" + + @event_logger.log_this + @expose("/") + def index(self, url_id: int) -> FlaskResponse: # pylint: disable=no-self-use + url = db.session.query(models.Url).get(url_id) + if url and url.url: + explore_url = "//superset/explore/?" + if url.url.startswith(explore_url): + explore_url += f"r={url_id}" + return redirect(explore_url[1:]) + + return redirect(url.url[1:]) + + flash("URL to nowhere...", "danger") + return redirect("/") + + @event_logger.log_this + @has_access_api + @expose("/shortner/", methods=["POST"]) + def shortner(self) -> FlaskResponse: # pylint: disable=no-self-use + url = request.form.get("data") + obj = models.Url(url=url) + db.session.add(obj) + db.session.commit() + return Response( + "{scheme}://{request.headers[Host]}/r/{obj.id}".format( + scheme=request.scheme, request=request, obj=obj + ), + mimetype="text/plain", + ) diff --git a/superset/views/utils.py b/superset/views/utils.py index 2a8b2bfac2..73c3a54c5c 100644 --- a/superset/views/utils.py +++ b/superset/views/utils.py @@ -14,35 +14,53 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import logging from collections import defaultdict from datetime import date -from typing import Any, DefaultDict, Dict, List, Optional, Set, Tuple +from typing import Any, Callable, DefaultDict, Dict, List, Optional, Set, Tuple, Union from urllib import parse +import msgpack +import pyarrow as pa import simplejson as json from flask import g, request +from flask_appbuilder.security.sqla import models as ab_models from flask_appbuilder.security.sqla.models import User import superset.models.core as models -from superset import app, db, is_feature_enabled +from superset import ( + app, + dataframe, + db, + is_feature_enabled, + result_set, + security_manager, +) from superset.connectors.connector_registry import ConnectorRegistry -from superset.exceptions import SupersetException +from superset.errors import ErrorLevel, SupersetError, SupersetErrorType +from superset.exceptions import SupersetException, SupersetSecurityException from superset.legacy import update_time_range +from superset.models.core import Database from superset.models.dashboard import Dashboard from superset.models.slice import Slice +from superset.models.sql_lab import Query from superset.typing import FormData from superset.utils.core import QueryStatus, TimeRangeEndpoint +from superset.utils.decorators import stats_timing from superset.viz import BaseViz +logger = logging.getLogger(__name__) +stats_logger = app.config["STATS_LOGGER"] + if is_feature_enabled("SIP_38_VIZ_REARCHITECTURE"): from superset import viz_sip38 as viz else: from superset import viz # type: ignore -FORM_DATA_KEY_BLACKLIST: List[str] = [] +REJECTED_FORM_DATA_KEYS: List[str] = [] if not app.config["ENABLE_JAVASCRIPT_CONTROLS"]: - FORM_DATA_KEY_BLACKLIST = ["js_tooltip", "js_onclick_href", "js_data_mutator"] + REJECTED_FORM_DATA_KEYS = ["js_tooltip", "js_onclick_href", "js_data_mutator"] def bootstrap_user_data(user: User, include_perms: bool = False) -> Dict[str, Any]: @@ -91,7 +109,7 @@ def get_permissions( def get_viz( - form_data: FormData, datasource_type: str, datasource_id: int, force: bool = False, + form_data: FormData, datasource_type: str, datasource_id: int, force: bool = False ) -> BaseViz: viz_type = form_data.get("viz_type", "table") datasource = ConnectorRegistry.get_datasource( @@ -129,7 +147,7 @@ def get_form_data( url_form_data.update(form_data) form_data = url_form_data - form_data = {k: v for k, v in form_data.items() if k not in FORM_DATA_KEY_BLACKLIST} + form_data = {k: v for k, v in form_data.items() if k not in REJECTED_FORM_DATA_KEYS} # When a slice_id is present, load from DB and override # the form_data from the DB with the other form_data provided @@ -160,7 +178,7 @@ def get_form_data( def get_datasource_info( - datasource_id: Optional[int], datasource_type: Optional[str], form_data: FormData, + datasource_id: Optional[int], datasource_type: Optional[str], form_data: FormData ) -> Tuple[int, Optional[str]]: """ Compatibility layer for handling of datasource info @@ -222,7 +240,7 @@ def apply_display_max_row_limit( def get_time_range_endpoints( - form_data: FormData, slc: Optional[Slice] = None, slice_id: Optional[int] = None, + form_data: FormData, slc: Optional[Slice] = None, slice_id: Optional[int] = None ) -> Optional[Tuple[TimeRangeEndpoint, TimeRangeEndpoint]]: """ Get the slice aware time range endpoints from the form-data falling back to the SQL @@ -361,3 +379,122 @@ def is_slice_in_container( ) return False + + +def is_owner(obj: Union[Dashboard, Slice], user: User) -> bool: + """ Check if user is owner of the slice """ + return obj and user in obj.owners + + +def check_datasource_perms( + _self: Any, + datasource_type: Optional[str] = None, + datasource_id: Optional[int] = None, +) -> None: + """ + Check if user can access a cached response from explore_json. + + This function takes `self` since it must have the same signature as the + the decorated method. + + :param datasource_type: The datasource type, i.e., 'druid' or 'table' + :param datasource_id: The datasource ID + :raises SupersetSecurityException: If the user cannot access the resource + """ + + form_data = get_form_data()[0] + + try: + datasource_id, datasource_type = get_datasource_info( + datasource_id, datasource_type, form_data + ) + except SupersetException as ex: + raise SupersetSecurityException( + SupersetError( + error_type=SupersetErrorType.FAILED_FETCHING_DATASOURCE_INFO_ERROR, + level=ErrorLevel.ERROR, + message=str(ex), + ) + ) + + if datasource_type is None: + raise SupersetSecurityException( + SupersetError( + error_type=SupersetErrorType.UNKNOWN_DATASOURCE_TYPE_ERROR, + level=ErrorLevel.ERROR, + message="Could not determine datasource type", + ) + ) + + viz_obj = get_viz( + datasource_type=datasource_type, + datasource_id=datasource_id, + form_data=form_data, + force=False, + ) + + security_manager.assert_viz_permission(viz_obj) + + +def check_slice_perms(_self: Any, slice_id: int) -> None: + """ + Check if user can access a cached response from slice_json. + + This function takes `self` since it must have the same signature as the + the decorated method. + """ + + form_data, slc = get_form_data(slice_id, use_slice_data=True) + + if slc: + viz_obj = get_viz( + datasource_type=slc.datasource.type, + datasource_id=slc.datasource.id, + form_data=form_data, + force=False, + ) + + security_manager.assert_viz_permission(viz_obj) + + +def _deserialize_results_payload( + payload: Union[bytes, str], query: Query, use_msgpack: Optional[bool] = False +) -> Dict[str, Any]: + logger.debug(f"Deserializing from msgpack: {use_msgpack}") + if use_msgpack: + with stats_timing( + "sqllab.query.results_backend_msgpack_deserialize", stats_logger + ): + ds_payload = msgpack.loads(payload, raw=False) + + with stats_timing("sqllab.query.results_backend_pa_deserialize", stats_logger): + pa_table = pa.deserialize(ds_payload["data"]) + + df = result_set.SupersetResultSet.convert_table_to_df(pa_table) + ds_payload["data"] = dataframe.df_to_records(df) or [] + + db_engine_spec = query.database.db_engine_spec + all_columns, data, expanded_columns = db_engine_spec.expand_data( + ds_payload["selected_columns"], ds_payload["data"] + ) + ds_payload.update( + {"data": data, "columns": all_columns, "expanded_columns": expanded_columns} + ) + + return ds_payload + else: + with stats_timing( + "sqllab.query.results_backend_json_deserialize", stats_logger + ): + return json.loads(payload) + + +def get_cta_schema_name( + database: Database, user: ab_models.User, schema: str, sql: str +) -> Optional[str]: + func: Optional[Callable[[Database, ab_models.User, str, str], str]] = app.config[ + "SQLLAB_CTAS_SCHEMA_NAME_FUNC" + ] + if not func: + return None + return func(database, user, schema, sql) diff --git a/tests/core_tests.py b/tests/core_tests.py index 641a639381..26fca602b1 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -37,6 +37,7 @@ from unittest import mock, skipUnless import pandas as pd import sqlalchemy as sqla +import superset.views.utils from tests.test_app import app from superset import ( dataframe, @@ -1106,7 +1107,7 @@ class CoreTests(SupersetTestCase): self.assertIsInstance(serialized_payload, str) query_mock = mock.Mock() - deserialized_payload = views._deserialize_results_payload( + deserialized_payload = superset.views.utils._deserialize_results_payload( serialized_payload, query_mock, use_new_deserialization ) @@ -1159,7 +1160,7 @@ class CoreTests(SupersetTestCase): query_mock = mock.Mock() query_mock.database.db_engine_spec.expand_data = expand_data - deserialized_payload = views._deserialize_results_payload( + deserialized_payload = superset.views.utils._deserialize_results_payload( serialized_payload, query_mock, use_new_deserialization ) df = results.to_pandas_df()