chore: add statsd support to base API and refactor (#22887)

This commit is contained in:
Daniel Vaz Gaspar 2023-01-27 17:52:08 +00:00 committed by GitHub
parent bed10a0e2b
commit d00ba15c78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 167 additions and 138 deletions

View File

@ -18,7 +18,7 @@ from typing import Any
from flask import current_app as app from flask import current_app as app
from flask.wrappers import Response from flask.wrappers import Response
from flask_appbuilder.api import BaseApi, expose, permission_name, protect, rison, safe from flask_appbuilder.api import expose, permission_name, protect, rison, safe
from flask_babel import lazy_gettext as _ from flask_babel import lazy_gettext as _
from superset.advanced_data_type.schemas import ( from superset.advanced_data_type.schemas import (
@ -27,12 +27,13 @@ from superset.advanced_data_type.schemas import (
) )
from superset.advanced_data_type.types import AdvancedDataTypeResponse from superset.advanced_data_type.types import AdvancedDataTypeResponse
from superset.extensions import event_logger from superset.extensions import event_logger
from superset.views.base_api import BaseSupersetApi
config = app.config config = app.config
ADVANCED_DATA_TYPES = config["ADVANCED_DATA_TYPES"] ADVANCED_DATA_TYPES = config["ADVANCED_DATA_TYPES"]
class AdvancedDataTypeRestApi(BaseApi): class AdvancedDataTypeRestApi(BaseSupersetApi):
""" """
Advanced Data Type Rest API Advanced Data Type Rest API
-Will return available AdvancedDataTypes when the /types endpoint is accessed -Will return available AdvancedDataTypes when the /types endpoint is accessed
@ -41,7 +42,6 @@ class AdvancedDataTypeRestApi(BaseApi):
""" """
allow_browser_login = True allow_browser_login = True
include_route_methods = {"get", "get_types"}
resource_name = "advanced_data_type" resource_name = "advanced_data_type"
class_permission_name = "AdvancedDataType" class_permission_name = "AdvancedDataType"

View File

@ -18,21 +18,19 @@ import logging
from flask import request, Response from flask import request, Response
from flask_appbuilder import expose from flask_appbuilder import expose
from flask_appbuilder.api import BaseApi, safe from flask_appbuilder.api import safe
from flask_appbuilder.security.decorators import permission_name, protect from flask_appbuilder.security.decorators import permission_name, protect
from superset.extensions import async_query_manager, event_logger from superset.extensions import async_query_manager, event_logger
from superset.utils.async_query_manager import AsyncQueryTokenException from superset.utils.async_query_manager import AsyncQueryTokenException
from superset.views.base_api import BaseSupersetApi
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class AsyncEventsRestApi(BaseApi): class AsyncEventsRestApi(BaseSupersetApi):
resource_name = "async_event" resource_name = "async_event"
allow_browser_login = True allow_browser_login = True
include_route_methods = {
"events",
}
@expose("/", methods=["GET"]) @expose("/", methods=["GET"])
@event_logger.log_this @event_logger.log_this

View File

@ -17,21 +17,21 @@
import logging import logging
from flask import Response from flask import Response
from flask_appbuilder.api import BaseApi, expose, protect, safe from flask_appbuilder.api import expose, protect, safe
from superset import conf from superset import conf
from superset.available_domains.schemas import AvailableDomainsSchema from superset.available_domains.schemas import AvailableDomainsSchema
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.extensions import event_logger from superset.extensions import event_logger
from superset.views.base_api import BaseSupersetApi, statsd_metrics
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class AvailableDomainsRestApi(BaseApi): class AvailableDomainsRestApi(BaseSupersetApi):
available_domains_schema = AvailableDomainsSchema() available_domains_schema = AvailableDomainsSchema()
method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP
include_route_methods = {RouteMethod.GET}
allow_browser_login = True allow_browser_login = True
class_permission_name = "AvailableDomains" class_permission_name = "AvailableDomains"
resource_name = "available_domains" resource_name = "available_domains"
@ -41,6 +41,7 @@ class AvailableDomainsRestApi(BaseApi):
@expose("/", methods=["GET"]) @expose("/", methods=["GET"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=True, log_to_statsd=True,

View File

@ -26,7 +26,7 @@ from sqlalchemy.exc import SQLAlchemyError
from superset.cachekeys.schemas import CacheInvalidationRequestSchema from superset.cachekeys.schemas import CacheInvalidationRequestSchema
from superset.connectors.sqla.models import SqlaTable from superset.connectors.sqla.models import SqlaTable
from superset.extensions import cache_manager, db, event_logger from superset.extensions import cache_manager, db, event_logger, stats_logger_manager
from superset.models.cache import CacheKey from superset.models.cache import CacheKey
from superset.views.base_api import BaseSupersetModelRestApi, statsd_metrics from superset.views.base_api import BaseSupersetModelRestApi, statsd_metrics
@ -117,7 +117,9 @@ class CacheRestApi(BaseSupersetModelRestApi):
) )
db.session.execute(delete_stmt) db.session.execute(delete_stmt)
db.session.commit() db.session.commit()
self.stats_logger.gauge("invalidated_cache", len(cache_keys)) stats_logger_manager.instance.gauge(
"invalidated_cache", len(cache_keys)
)
logger.info( logger.info(
"Invalidated %s cache records for %s datasources", "Invalidated %s cache records for %s datasources",
len(cache_keys), len(cache_keys),

View File

@ -17,10 +17,10 @@
import logging import logging
from flask import request, Response from flask import request, Response
from flask_appbuilder.api import BaseApi, expose, protect, safe from flask_appbuilder.api import expose, protect, safe
from marshmallow import ValidationError from marshmallow import ValidationError
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.dashboards.commands.exceptions import ( from superset.dashboards.commands.exceptions import (
DashboardAccessDeniedError, DashboardAccessDeniedError,
DashboardNotFoundError, DashboardNotFoundError,
@ -33,20 +33,14 @@ from superset.dashboards.permalink.exceptions import DashboardPermalinkInvalidSt
from superset.dashboards.permalink.schemas import DashboardPermalinkPostSchema from superset.dashboards.permalink.schemas import DashboardPermalinkPostSchema
from superset.extensions import event_logger from superset.extensions import event_logger
from superset.key_value.exceptions import KeyValueAccessDeniedError from superset.key_value.exceptions import KeyValueAccessDeniedError
from superset.views.base_api import requires_json from superset.views.base_api import BaseSupersetApi, requires_json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class DashboardPermalinkRestApi(BaseApi): class DashboardPermalinkRestApi(BaseSupersetApi):
add_model_schema = DashboardPermalinkPostSchema() add_model_schema = DashboardPermalinkPostSchema()
method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP
include_route_methods = {
RouteMethod.POST,
RouteMethod.PUT,
RouteMethod.GET,
RouteMethod.DELETE,
}
allow_browser_login = True allow_browser_login = True
class_permission_name = "DashboardPermalinkRestApi" class_permission_name = "DashboardPermalinkRestApi"
resource_name = "dashboard" resource_name = "dashboard"

View File

@ -21,6 +21,7 @@ from typing import Any, Callable, Optional
from flask import g from flask import g
from flask_babel import lazy_gettext as _ from flask_babel import lazy_gettext as _
from superset.extensions import stats_logger_manager
from superset.models.core import Database from superset.models.core import Database
from superset.sql_parse import Table from superset.sql_parse import Table
from superset.utils.core import parse_js_uri_path_item from superset.utils.core import parse_js_uri_path_item
@ -46,14 +47,14 @@ def check_datasource_access(f: Callable[..., Any]) -> Callable[..., Any]:
return self.response_422(message=_("Table name undefined")) return self.response_422(message=_("Table name undefined"))
database: Database = self.datamodel.get(pk) database: Database = self.datamodel.get(pk)
if not database: if not database:
self.stats_logger.incr( stats_logger_manager.instance.incr(
f"database_not_found_{self.__class__.__name__}.select_star" f"database_not_found_{self.__class__.__name__}.select_star"
) )
return self.response_404() return self.response_404()
if not self.appbuilder.sm.can_access_table( if not self.appbuilder.sm.can_access_table(
database, Table(table_name_parsed, schema_name_parsed) database, Table(table_name_parsed, schema_name_parsed)
): ):
self.stats_logger.incr( stats_logger_manager.instance.incr(
f"permisssion_denied_{self.__class__.__name__}.select_star" f"permisssion_denied_{self.__class__.__name__}.select_star"
) )
logger.warning( logger.warning(

View File

@ -17,10 +17,10 @@
import logging import logging
from flask import g, request, Response from flask import g, request, Response
from flask_appbuilder.api import BaseApi, expose, protect, safe from flask_appbuilder.api import expose, protect, safe
from superset.charts.commands.exceptions import ChartNotFoundError from superset.charts.commands.exceptions import ChartNotFoundError
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.explore.commands.get import GetExploreCommand from superset.explore.commands.get import GetExploreCommand
from superset.explore.commands.parameters import CommandParameters from superset.explore.commands.parameters import CommandParameters
from superset.explore.exceptions import DatasetAccessDeniedError, WrongEndpointError from superset.explore.exceptions import DatasetAccessDeniedError, WrongEndpointError
@ -31,13 +31,13 @@ from superset.temporary_cache.commands.exceptions import (
TemporaryCacheAccessDeniedError, TemporaryCacheAccessDeniedError,
TemporaryCacheResourceNotFoundError, TemporaryCacheResourceNotFoundError,
) )
from superset.views.base_api import BaseSupersetApi, statsd_metrics
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class ExploreRestApi(BaseApi): class ExploreRestApi(BaseSupersetApi):
method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP
include_route_methods = {RouteMethod.GET}
allow_browser_login = True allow_browser_login = True
class_permission_name = "Explore" class_permission_name = "Explore"
resource_name = "explore" resource_name = "explore"
@ -47,6 +47,7 @@ class ExploreRestApi(BaseApi):
@expose("/", methods=["GET"]) @expose("/", methods=["GET"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=True, log_to_statsd=True,

View File

@ -17,10 +17,10 @@
import logging import logging
from flask import request, Response from flask import request, Response
from flask_appbuilder.api import BaseApi, expose, protect, safe from flask_appbuilder.api import expose, protect, safe
from marshmallow import ValidationError from marshmallow import ValidationError
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.explore.form_data.commands.create import CreateFormDataCommand from superset.explore.form_data.commands.create import CreateFormDataCommand
from superset.explore.form_data.commands.delete import DeleteFormDataCommand from superset.explore.form_data.commands.delete import DeleteFormDataCommand
from superset.explore.form_data.commands.get import GetFormDataCommand from superset.explore.form_data.commands.get import GetFormDataCommand
@ -32,21 +32,15 @@ from superset.temporary_cache.commands.exceptions import (
TemporaryCacheAccessDeniedError, TemporaryCacheAccessDeniedError,
TemporaryCacheResourceNotFoundError, TemporaryCacheResourceNotFoundError,
) )
from superset.views.base_api import requires_json from superset.views.base_api import BaseSupersetApi, requires_json, statsd_metrics
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class ExploreFormDataRestApi(BaseApi): class ExploreFormDataRestApi(BaseSupersetApi):
add_model_schema = FormDataPostSchema() add_model_schema = FormDataPostSchema()
edit_model_schema = FormDataPutSchema() edit_model_schema = FormDataPutSchema()
method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP
include_route_methods = {
RouteMethod.POST,
RouteMethod.PUT,
RouteMethod.GET,
RouteMethod.DELETE,
}
allow_browser_login = True allow_browser_login = True
class_permission_name = "ExploreFormDataRestApi" class_permission_name = "ExploreFormDataRestApi"
resource_name = "explore" resource_name = "explore"
@ -56,6 +50,7 @@ class ExploreFormDataRestApi(BaseApi):
@expose("/form_data", methods=["POST"]) @expose("/form_data", methods=["POST"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.post", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.post",
log_to_statsd=False, log_to_statsd=False,
@ -120,6 +115,7 @@ class ExploreFormDataRestApi(BaseApi):
@expose("/form_data/<string:key>", methods=["PUT"]) @expose("/form_data/<string:key>", methods=["PUT"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.put", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.put",
log_to_statsd=True, log_to_statsd=True,
@ -193,6 +189,7 @@ class ExploreFormDataRestApi(BaseApi):
@expose("/form_data/<string:key>", methods=["GET"]) @expose("/form_data/<string:key>", methods=["GET"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=True, log_to_statsd=True,
@ -244,6 +241,7 @@ class ExploreFormDataRestApi(BaseApi):
@expose("/form_data/<string:key>", methods=["DELETE"]) @expose("/form_data/<string:key>", methods=["DELETE"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.delete", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.delete",
log_to_statsd=True, log_to_statsd=True,

View File

@ -17,14 +17,14 @@
import logging import logging
from flask import request, Response from flask import request, Response
from flask_appbuilder.api import BaseApi, expose, protect, safe from flask_appbuilder.api import expose, protect, safe
from marshmallow import ValidationError from marshmallow import ValidationError
from superset.charts.commands.exceptions import ( from superset.charts.commands.exceptions import (
ChartAccessDeniedError, ChartAccessDeniedError,
ChartNotFoundError, ChartNotFoundError,
) )
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.datasets.commands.exceptions import ( from superset.datasets.commands.exceptions import (
DatasetAccessDeniedError, DatasetAccessDeniedError,
DatasetNotFoundError, DatasetNotFoundError,
@ -35,20 +35,14 @@ from superset.explore.permalink.exceptions import ExplorePermalinkInvalidStateEr
from superset.explore.permalink.schemas import ExplorePermalinkPostSchema from superset.explore.permalink.schemas import ExplorePermalinkPostSchema
from superset.extensions import event_logger from superset.extensions import event_logger
from superset.key_value.exceptions import KeyValueAccessDeniedError from superset.key_value.exceptions import KeyValueAccessDeniedError
from superset.views.base_api import requires_json from superset.views.base_api import BaseSupersetApi, requires_json, statsd_metrics
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class ExplorePermalinkRestApi(BaseApi): class ExplorePermalinkRestApi(BaseSupersetApi):
add_model_schema = ExplorePermalinkPostSchema() add_model_schema = ExplorePermalinkPostSchema()
method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP
include_route_methods = {
RouteMethod.POST,
RouteMethod.PUT,
RouteMethod.GET,
RouteMethod.DELETE,
}
allow_browser_login = True allow_browser_login = True
class_permission_name = "ExplorePermalinkRestApi" class_permission_name = "ExplorePermalinkRestApi"
resource_name = "explore" resource_name = "explore"
@ -58,6 +52,7 @@ class ExplorePermalinkRestApi(BaseApi):
@expose("/permalink", methods=["POST"]) @expose("/permalink", methods=["POST"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.post", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.post",
log_to_statsd=False, log_to_statsd=False,
@ -118,6 +113,7 @@ class ExplorePermalinkRestApi(BaseApi):
@expose("/permalink/<string:key>", methods=["GET"]) @expose("/permalink/<string:key>", methods=["GET"])
@protect() @protect()
@safe @safe
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=False, log_to_statsd=False,

View File

@ -16,7 +16,6 @@
# under the License. # under the License.
import json import json
import os import os
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional from typing import Any, Callable, Dict, List, Optional
import celery import celery
@ -29,6 +28,7 @@ from flask_wtf.csrf import CSRFProtect
from werkzeug.local import LocalProxy from werkzeug.local import LocalProxy
from superset.extensions.ssh import SSHManagerFactory from superset.extensions.ssh import SSHManagerFactory
from superset.extensions.stats_logger import BaseStatsLoggerManager
from superset.utils.async_query_manager import AsyncQueryManager from superset.utils.async_query_manager import AsyncQueryManager
from superset.utils.cache_manager import CacheManager from superset.utils.cache_manager import CacheManager
from superset.utils.encrypt import EncryptedFieldFactory from superset.utils.encrypt import EncryptedFieldFactory
@ -127,5 +127,6 @@ migrate = Migrate()
profiling = ProfilingExtension() profiling = ProfilingExtension()
results_backend_manager = ResultsBackendManager() results_backend_manager = ResultsBackendManager()
security_manager = LocalProxy(lambda: appbuilder.sm) security_manager = LocalProxy(lambda: appbuilder.sm)
talisman = Talisman()
ssh_manager_factory = SSHManagerFactory() ssh_manager_factory = SSHManagerFactory()
stats_logger_manager = BaseStatsLoggerManager()
talisman = Talisman()

View File

@ -0,0 +1,31 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from flask import Flask
from superset.stats_logger import BaseStatsLogger
class BaseStatsLoggerManager:
def __init__(self) -> None:
self._stats_logger = BaseStatsLogger()
def init_app(self, app: Flask) -> None:
self._stats_logger = app.config["STATS_LOGGER"]
@property
def instance(self) -> BaseStatsLogger:
return self._stats_logger

View File

@ -20,7 +20,7 @@ from io import BytesIO
from zipfile import is_zipfile, ZipFile from zipfile import is_zipfile, ZipFile
from flask import request, Response, send_file from flask import request, Response, send_file
from flask_appbuilder.api import BaseApi, expose, protect from flask_appbuilder.api import expose, protect
from superset.commands.export.assets import ExportAssetsCommand from superset.commands.export.assets import ExportAssetsCommand
from superset.commands.importers.exceptions import ( from superset.commands.importers.exceptions import (
@ -30,10 +30,10 @@ from superset.commands.importers.exceptions import (
from superset.commands.importers.v1.assets import ImportAssetsCommand from superset.commands.importers.v1.assets import ImportAssetsCommand
from superset.commands.importers.v1.utils import get_contents_from_bundle from superset.commands.importers.v1.utils import get_contents_from_bundle
from superset.extensions import event_logger from superset.extensions import event_logger
from superset.views.base_api import requires_form_data from superset.views.base_api import BaseSupersetApi, requires_form_data, statsd_metrics
class ImportExportRestApi(BaseApi): class ImportExportRestApi(BaseSupersetApi):
""" """
API for exporting all assets or importing them. API for exporting all assets or importing them.
""" """
@ -44,6 +44,7 @@ class ImportExportRestApi(BaseApi):
@expose("/export/", methods=["GET"]) @expose("/export/", methods=["GET"])
@protect() @protect()
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.export", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.export",
log_to_statsd=False, log_to_statsd=False,
@ -92,6 +93,7 @@ class ImportExportRestApi(BaseApi):
@expose("/import/", methods=["POST"]) @expose("/import/", methods=["POST"])
@protect() @protect()
@statsd_metrics
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.import_", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.import_",
log_to_statsd=False, log_to_statsd=False,

View File

@ -46,6 +46,7 @@ from superset.extensions import (
profiling, profiling,
results_backend_manager, results_backend_manager,
ssh_manager_factory, ssh_manager_factory,
stats_logger_manager,
talisman, talisman,
) )
from superset.security import SupersetSecurityManager from superset.security import SupersetSecurityManager
@ -419,6 +420,7 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
self.configure_auth_provider() self.configure_auth_provider()
self.configure_async_queries() self.configure_async_queries()
self.configure_ssh_manager() self.configure_ssh_manager()
self.configure_stats_manager()
# Hook that provides administrators a handle on the Flask APP # Hook that provides administrators a handle on the Flask APP
# after initialization # after initialization
@ -479,6 +481,9 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
def configure_ssh_manager(self) -> None: def configure_ssh_manager(self) -> None:
ssh_manager_factory.init_app(self.superset_app) ssh_manager_factory.init_app(self.superset_app)
def configure_stats_manager(self) -> None:
stats_logger_manager.init_app(self.superset_app)
def setup_event_logger(self) -> None: def setup_event_logger(self) -> None:
_event_logger["event_logger"] = get_event_logger_from_cfg_value( _event_logger["event_logger"] = get_event_logger_from_cfg_value(
self.superset_app.config.get("EVENT_LOGGER", DBEventLogger()) self.superset_app.config.get("EVENT_LOGGER", DBEventLogger())

View File

@ -19,7 +19,7 @@ from typing import Any, Dict
from flask import request, Response from flask import request, Response
from flask_appbuilder import expose from flask_appbuilder import expose
from flask_appbuilder.api import BaseApi, safe from flask_appbuilder.api import safe
from flask_appbuilder.security.decorators import permission_name, protect from flask_appbuilder.security.decorators import permission_name, protect
from flask_wtf.csrf import generate_csrf from flask_wtf.csrf import generate_csrf
from marshmallow import EXCLUDE, fields, post_load, Schema, ValidationError from marshmallow import EXCLUDE, fields, post_load, Schema, ValidationError
@ -30,6 +30,7 @@ from superset.embedded_dashboard.commands.exceptions import (
) )
from superset.extensions import event_logger from superset.extensions import event_logger
from superset.security.guest_token import GuestTokenResourceType from superset.security.guest_token import GuestTokenResourceType
from superset.views.base_api import BaseSupersetApi, statsd_metrics
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -76,7 +77,7 @@ class GuestTokenCreateSchema(PermissiveSchema):
guest_token_create_schema = GuestTokenCreateSchema() guest_token_create_schema = GuestTokenCreateSchema()
class SecurityRestApi(BaseApi): class SecurityRestApi(BaseSupersetApi):
resource_name = "security" resource_name = "security"
allow_browser_login = True allow_browser_login = True
openapi_spec_tag = "Security" openapi_spec_tag = "Security"
@ -85,6 +86,7 @@ class SecurityRestApi(BaseApi):
@event_logger.log_this @event_logger.log_this
@protect() @protect()
@safe @safe
@statsd_metrics
@permission_name("read") @permission_name("read")
def csrf_token(self) -> Response: def csrf_token(self) -> Response:
""" """
@ -114,6 +116,7 @@ class SecurityRestApi(BaseApi):
@event_logger.log_this @event_logger.log_this
@protect() @protect()
@safe @safe
@statsd_metrics
@permission_name("grant_guest_token") @permission_name("grant_guest_token")
def guest_token(self) -> Response: def guest_token(self) -> Response:
"""Response """Response

View File

@ -21,7 +21,6 @@ from typing import Any
from apispec import APISpec from apispec import APISpec
from apispec.exceptions import DuplicateComponentNameError from apispec.exceptions import DuplicateComponentNameError
from flask import request, Response from flask import request, Response
from flask_appbuilder.api import BaseApi
from marshmallow import ValidationError from marshmallow import ValidationError
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod
@ -34,12 +33,12 @@ from superset.temporary_cache.schemas import (
TemporaryCachePostSchema, TemporaryCachePostSchema,
TemporaryCachePutSchema, TemporaryCachePutSchema,
) )
from superset.views.base_api import requires_json from superset.views.base_api import BaseSupersetApi, requires_json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class TemporaryCacheRestApi(BaseApi, ABC): class TemporaryCacheRestApi(BaseSupersetApi, ABC):
add_model_schema = TemporaryCachePostSchema() add_model_schema = TemporaryCachePostSchema()
edit_model_schema = TemporaryCachePutSchema() edit_model_schema = TemporaryCachePutSchema()
method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP

View File

@ -42,6 +42,7 @@ from flask_appbuilder.const import API_URI_RIS_KEY
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from typing_extensions import Literal from typing_extensions import Literal
from superset.extensions import stats_logger_manager
from superset.utils.core import get_user_id, LoggerLevel from superset.utils.core import get_user_id, LoggerLevel
if TYPE_CHECKING: if TYPE_CHECKING:
@ -194,7 +195,7 @@ class AbstractEventLogger(ABC):
slice_id = 0 slice_id = 0
if log_to_statsd: if log_to_statsd:
self.stats_logger.incr(action) stats_logger_manager.instance.incr(action)
try: try:
# bulk insert # bulk insert
@ -283,10 +284,6 @@ class AbstractEventLogger(ABC):
"""Decorator that instrument `update_log_payload` to kwargs""" """Decorator that instrument `update_log_payload` to kwargs"""
return self._wrapper(f, allow_extra_payload=True) return self._wrapper(f, allow_extra_payload=True)
@property
def stats_logger(self) -> BaseStatsLogger:
return current_app.config["STATS_LOGGER"]
def get_event_logger_from_cfg_value(cfg_value: Any) -> AbstractEventLogger: def get_event_logger_from_cfg_value(cfg_value: Any) -> AbstractEventLogger:
""" """

View File

@ -14,13 +14,15 @@
# KIND, either express or implied. See the License for the # KIND, either express or implied. See the License for the
# specific language governing permissions and limitations # specific language governing permissions and limitations
# under the License. # under the License.
from __future__ import annotations
import functools import functools
import logging import logging
from typing import Any, Callable, cast, Dict, List, Optional, Set, Tuple, Type, Union from typing import Any, Callable, cast, Dict, List, Optional, Set, Tuple, Type, Union
from flask import Blueprint, request, Response from flask import request, Response
from flask_appbuilder import AppBuilder, Model, ModelRestApi from flask_appbuilder import Model, ModelRestApi
from flask_appbuilder.api import expose, protect, rison, safe from flask_appbuilder.api import BaseApi, expose, protect, rison, safe
from flask_appbuilder.models.filters import BaseFilter, Filters from flask_appbuilder.models.filters import BaseFilter, Filters
from flask_appbuilder.models.sqla.filters import FilterStartsWith from flask_appbuilder.models.sqla.filters import FilterStartsWith
from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_appbuilder.models.sqla.interface import SQLAInterface
@ -30,13 +32,12 @@ from sqlalchemy import and_, distinct, func
from sqlalchemy.orm.query import Query from sqlalchemy.orm.query import Query
from superset.exceptions import InvalidPayloadFormatError from superset.exceptions import InvalidPayloadFormatError
from superset.extensions import db, event_logger, security_manager from superset.extensions import db, event_logger, security_manager, stats_logger_manager
from superset.models.core import FavStar from superset.models.core import FavStar
from superset.models.dashboard import Dashboard from superset.models.dashboard import Dashboard
from superset.models.slice import Slice from superset.models.slice import Slice
from superset.schemas import error_payload_content from superset.schemas import error_payload_content
from superset.sql_lab import Query as SqllabQuery from superset.sql_lab import Query as SqllabQuery
from superset.stats_logger import BaseStatsLogger
from superset.superset_typing import FlaskResponse from superset.superset_typing import FlaskResponse
from superset.utils.core import get_user_id, time_function from superset.utils.core import get_user_id, time_function
from superset.views.base import handle_api_exception from superset.views.base import handle_api_exception
@ -91,7 +92,7 @@ def requires_form_data(f: Callable[..., Any]) -> Callable[..., Any]:
Require 'multipart/form-data' as request MIME type Require 'multipart/form-data' as request MIME type
""" """
def wraps(self: "BaseSupersetModelRestApi", *args: Any, **kwargs: Any) -> Response: def wraps(self: BaseSupersetApiMixin, *args: Any, **kwargs: Any) -> Response:
if not request.mimetype == "multipart/form-data": if not request.mimetype == "multipart/form-data":
raise InvalidPayloadFormatError( raise InvalidPayloadFormatError(
message="Request MIME type is not 'multipart/form-data'" message="Request MIME type is not 'multipart/form-data'"
@ -106,14 +107,15 @@ def statsd_metrics(f: Callable[..., Any]) -> Callable[..., Any]:
Handle sending all statsd metrics from the REST API Handle sending all statsd metrics from the REST API
""" """
def wraps(self: "BaseSupersetModelRestApi", *args: Any, **kwargs: Any) -> Response: def wraps(self: BaseSupersetApiMixin, *args: Any, **kwargs: Any) -> Response:
func_name = f.__name__
try: try:
duration, response = time_function(f, self, *args, **kwargs) duration, response = time_function(f, self, *args, **kwargs)
except Exception as ex: except Exception as ex:
self.incr_stats("error", f.__name__) self.incr_stats("error", func_name)
raise ex raise ex
self.send_stats_metrics(response, f.__name__, duration) self.send_stats_metrics(response, func_name, duration)
return response return response
return functools.update_wrapper(wraps, f) return functools.update_wrapper(wraps, f)
@ -155,12 +157,68 @@ class BaseFavoriteFilter(BaseFilter): # pylint: disable=too-few-public-methods
return query.filter(and_(~self.model.id.in_(users_favorite_query))) return query.filter(and_(~self.model.id.in_(users_favorite_query)))
class BaseSupersetModelRestApi(ModelRestApi): class BaseSupersetApiMixin:
csrf_exempt = False
responses = {
"400": {"description": "Bad request", "content": error_payload_content},
"401": {"description": "Unauthorized", "content": error_payload_content},
"403": {"description": "Forbidden", "content": error_payload_content},
"404": {"description": "Not found", "content": error_payload_content},
"422": {
"description": "Could not process entity",
"content": error_payload_content,
},
"500": {"description": "Fatal error", "content": error_payload_content},
}
def incr_stats(self, action: str, func_name: str) -> None:
"""
Proxy function for statsd.incr to impose a key structure for REST API's
:param action: String with an action name eg: error, success
:param func_name: The function name
"""
stats_logger_manager.instance.incr(
f"{self.__class__.__name__}.{func_name}.{action}"
)
def timing_stats(self, action: str, func_name: str, value: float) -> None:
"""
Proxy function for statsd.incr to impose a key structure for REST API's
:param action: String with an action name eg: error, success
:param func_name: The function name
:param value: A float with the time it took for the endpoint to execute
"""
stats_logger_manager.instance.timing(
f"{self.__class__.__name__}.{func_name}.{action}", value
)
def send_stats_metrics(
self, response: Response, key: str, time_delta: Optional[float] = None
) -> None:
"""
Helper function to handle sending statsd metrics
:param response: flask response object, will evaluate if it was an error
:param key: The function name
:param time_delta: Optional time it took for the endpoint to execute
"""
if 200 <= response.status_code < 400:
self.incr_stats("success", key)
else:
self.incr_stats("error", key)
if time_delta:
self.timing_stats("time", key, time_delta)
class BaseSupersetApi(BaseApi, BaseSupersetApiMixin):
...
class BaseSupersetModelRestApi(ModelRestApi, BaseSupersetApiMixin):
""" """
Extends FAB's ModelResApi to implement specific superset generic functionality Extends FAB's ModelResApi to implement specific superset generic functionality
""" """
csrf_exempt = False
method_permission_name = { method_permission_name = {
"bulk_delete": "delete", "bulk_delete": "delete",
"data": "list", "data": "list",
@ -246,22 +304,8 @@ class BaseSupersetModelRestApi(ModelRestApi):
list_columns: List[str] list_columns: List[str]
show_columns: List[str] show_columns: List[str]
responses = {
"400": {"description": "Bad request", "content": error_payload_content},
"401": {"description": "Unauthorized", "content": error_payload_content},
"403": {"description": "Forbidden", "content": error_payload_content},
"404": {"description": "Not found", "content": error_payload_content},
"422": {
"description": "Could not process entity",
"content": error_payload_content,
},
"500": {"description": "Fatal error", "content": error_payload_content},
}
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
# Setup statsd
self.stats_logger = BaseStatsLogger()
# Add base API spec base query parameter schemas # Add base API spec base query parameter schemas
if self.apispec_parameter_schemas is None: # type: ignore if self.apispec_parameter_schemas is None: # type: ignore
self.apispec_parameter_schemas = {} self.apispec_parameter_schemas = {}
@ -273,12 +317,6 @@ class BaseSupersetModelRestApi(ModelRestApi):
DistincResponseSchema, DistincResponseSchema,
) )
def create_blueprint(
self, appbuilder: AppBuilder, *args: Any, **kwargs: Any
) -> Blueprint:
self.stats_logger = self.appbuilder.get_app.config["STATS_LOGGER"]
return super().create_blueprint(appbuilder, *args, **kwargs)
def _init_properties(self) -> None: def _init_properties(self) -> None:
""" """
Lock down initial not configured REST API columns. We want to just expose Lock down initial not configured REST API columns. We want to just expose
@ -372,44 +410,6 @@ class BaseSupersetModelRestApi(ModelRestApi):
extra_rows = db.session.query(datamodel.obj).filter(pk_col.in_(ids)).all() extra_rows = db.session.query(datamodel.obj).filter(pk_col.in_(ids)).all()
result += self._get_result_from_rows(datamodel, extra_rows, column_name) result += self._get_result_from_rows(datamodel, extra_rows, column_name)
def incr_stats(self, action: str, func_name: str) -> None:
"""
Proxy function for statsd.incr to impose a key structure for REST API's
:param action: String with an action name eg: error, success
:param func_name: The function name
"""
self.stats_logger.incr(f"{self.__class__.__name__}.{func_name}.{action}")
def timing_stats(self, action: str, func_name: str, value: float) -> None:
"""
Proxy function for statsd.incr to impose a key structure for REST API's
:param action: String with an action name eg: error, success
:param func_name: The function name
:param value: A float with the time it took for the endpoint to execute
"""
self.stats_logger.timing(
f"{self.__class__.__name__}.{func_name}.{action}", value
)
def send_stats_metrics(
self, response: Response, key: str, time_delta: Optional[float] = None
) -> None:
"""
Helper function to handle sending statsd metrics
:param response: flask response object, will evaluate if it was an error
:param key: The function name
:param time_delta: Optional time it took for the endpoint to execute
"""
if 200 <= response.status_code < 400:
self.incr_stats("success", key)
else:
self.incr_stats("error", key)
if time_delta:
self.timing_stats("time", key, time_delta)
@event_logger.log_this_with_context( @event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.info", action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.info",
object_ref=False, object_ref=False,

View File

@ -15,17 +15,17 @@
# specific language governing permissions and limitations # specific language governing permissions and limitations
# under the License. # under the License.
from flask import g, Response from flask import g, Response
from flask_appbuilder.api import BaseApi, expose, safe from flask_appbuilder.api import expose, safe
from flask_jwt_extended.exceptions import NoAuthorizationError from flask_jwt_extended.exceptions import NoAuthorizationError
from superset.views.base_api import BaseSupersetApi
from superset.views.users.schemas import UserResponseSchema
from superset.views.utils import bootstrap_user_data from superset.views.utils import bootstrap_user_data
from .schemas import UserResponseSchema
user_response_schema = UserResponseSchema() user_response_schema = UserResponseSchema()
class CurrentUserRestApi(BaseApi): class CurrentUserRestApi(BaseSupersetApi):
"""An api to get information about the current user""" """An api to get information about the current user"""
resource_name = "me" resource_name = "me"