2020-07-28 19:48:42 -04:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
# distributed with this work for additional information
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
# under the License.
|
|
|
|
"""Unit tests for alerting in Superset"""
|
|
|
|
import logging
|
2020-09-01 16:36:02 -04:00
|
|
|
from typing import Optional
|
2020-07-28 19:48:42 -04:00
|
|
|
from unittest.mock import patch
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
from superset import db
|
2020-09-01 16:36:02 -04:00
|
|
|
from superset.exceptions import SupersetException
|
|
|
|
from superset.models.alerts import (
|
|
|
|
Alert,
|
|
|
|
AlertLog,
|
|
|
|
SQLObservation,
|
|
|
|
SQLObserver,
|
|
|
|
Validator,
|
|
|
|
)
|
2020-07-28 19:48:42 -04:00
|
|
|
from superset.models.slice import Slice
|
2020-09-01 16:36:02 -04:00
|
|
|
from superset.tasks.alerts.observer import observe
|
|
|
|
from superset.tasks.alerts.validator import (
|
|
|
|
check_validator,
|
|
|
|
not_null_validator,
|
|
|
|
operator_validator,
|
|
|
|
)
|
2020-08-11 14:15:31 -04:00
|
|
|
from superset.tasks.schedules import (
|
2020-09-01 16:36:02 -04:00
|
|
|
AlertState,
|
2020-08-11 14:15:31 -04:00
|
|
|
deliver_alert,
|
2020-09-01 16:36:02 -04:00
|
|
|
evaluate_alert,
|
|
|
|
validate_observations,
|
2020-08-11 14:15:31 -04:00
|
|
|
)
|
2020-07-28 19:48:42 -04:00
|
|
|
from superset.utils import core as utils
|
|
|
|
from tests.test_app import app
|
2020-08-11 14:15:31 -04:00
|
|
|
from tests.utils import read_fixture
|
2020-07-28 19:48:42 -04:00
|
|
|
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
2020-08-06 18:33:48 -04:00
|
|
|
@pytest.yield_fixture(scope="module")
|
|
|
|
def setup_database():
|
2020-07-28 19:48:42 -04:00
|
|
|
with app.app_context():
|
2020-09-01 16:36:02 -04:00
|
|
|
example_database = utils.get_example_database()
|
|
|
|
example_database.get_sqla_engine().execute(
|
|
|
|
"CREATE TABLE test_table AS SELECT 1 as first, 2 as second"
|
|
|
|
)
|
|
|
|
example_database.get_sqla_engine().execute(
|
|
|
|
"INSERT INTO test_table (first, second) VALUES (3, 4)"
|
|
|
|
)
|
|
|
|
|
|
|
|
no_observer_alert = Alert(crontab="* * * * *", label="No Observer")
|
|
|
|
db.session.add(no_observer_alert)
|
2020-08-06 18:33:48 -04:00
|
|
|
db.session.commit()
|
|
|
|
yield db.session
|
2020-07-31 02:07:56 -04:00
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
db.session.query(SQLObservation).delete()
|
|
|
|
db.session.query(SQLObserver).delete()
|
|
|
|
db.session.query(Validator).delete()
|
2020-07-28 19:48:42 -04:00
|
|
|
db.session.query(AlertLog).delete()
|
|
|
|
db.session.query(Alert).delete()
|
|
|
|
|
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
def create_alert(
|
|
|
|
dbsession,
|
|
|
|
sql: str,
|
|
|
|
validator_type: Optional[str] = None,
|
|
|
|
validator_config: Optional[str] = None,
|
|
|
|
) -> Alert:
|
|
|
|
alert = Alert(
|
|
|
|
label="test_alert",
|
|
|
|
active=True,
|
|
|
|
crontab="* * * * *",
|
|
|
|
slice_id=dbsession.query(Slice).all()[0].id,
|
|
|
|
recipients="recipient1@superset.com",
|
|
|
|
slack_channel="#test_channel",
|
|
|
|
)
|
|
|
|
dbsession.add(alert)
|
|
|
|
dbsession.commit()
|
|
|
|
|
|
|
|
sql_observer = SQLObserver(
|
|
|
|
sql=sql, alert_id=alert.id, database_id=utils.get_example_database().id,
|
|
|
|
)
|
|
|
|
|
|
|
|
if validator_type and validator_config:
|
|
|
|
validator = Validator(
|
|
|
|
validator_type=validator_type, config=validator_config, alert_id=alert.id,
|
|
|
|
)
|
|
|
|
|
|
|
|
dbsession.add(validator)
|
|
|
|
|
|
|
|
dbsession.add(sql_observer)
|
|
|
|
dbsession.commit()
|
|
|
|
return alert
|
|
|
|
|
|
|
|
|
|
|
|
def test_alert_observer(setup_database):
|
|
|
|
dbsession = setup_database
|
|
|
|
|
|
|
|
# Test SQLObserver with int SQL return
|
|
|
|
alert1 = create_alert(dbsession, "SELECT 55")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert1.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert1.sql_observer[0].observations[-1].value == 55.0
|
|
|
|
assert alert1.sql_observer[0].observations[-1].error_msg is None
|
|
|
|
|
|
|
|
# Test SQLObserver with double SQL return
|
|
|
|
alert2 = create_alert(dbsession, "SELECT 30.0 as wage")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert2.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert2.sql_observer[0].observations[-1].value == 30.0
|
|
|
|
assert alert2.sql_observer[0].observations[-1].error_msg is None
|
|
|
|
|
|
|
|
# Test SQLObserver with NULL result
|
|
|
|
alert3 = create_alert(dbsession, "SELECT null as null_result")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert3.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert3.sql_observer[0].observations[-1].value is None
|
|
|
|
assert alert3.sql_observer[0].observations[-1].error_msg is None
|
|
|
|
|
2020-09-21 11:48:02 -04:00
|
|
|
# Test SQLObserver with empty SQL return, expected
|
2020-09-01 16:36:02 -04:00
|
|
|
alert4 = create_alert(dbsession, "SELECT first FROM test_table WHERE first = -1")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert4.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert4.sql_observer[0].observations[-1].value is None
|
2020-09-21 11:48:02 -04:00
|
|
|
assert alert4.sql_observer[0].observations[-1].error_msg is None
|
2020-09-01 16:36:02 -04:00
|
|
|
|
|
|
|
# Test SQLObserver with str result
|
|
|
|
alert5 = create_alert(dbsession, "SELECT 'test_string' as string_value")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert5.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert5.sql_observer[0].observations[-1].value is None
|
|
|
|
assert alert5.sql_observer[0].observations[-1].error_msg is not None
|
|
|
|
|
|
|
|
# Test SQLObserver with two row result
|
|
|
|
alert6 = create_alert(dbsession, "SELECT first FROM test_table")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert6.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert6.sql_observer[0].observations[-1].value is None
|
|
|
|
assert alert6.sql_observer[0].observations[-1].error_msg is not None
|
|
|
|
|
|
|
|
# Test SQLObserver with two column result
|
|
|
|
alert7 = create_alert(
|
|
|
|
dbsession, "SELECT first, second FROM test_table WHERE first = 1"
|
|
|
|
)
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert7.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert7.sql_observer[0].observations[-1].value is None
|
|
|
|
assert alert7.sql_observer[0].observations[-1].error_msg is not None
|
|
|
|
|
2020-09-24 13:29:44 -04:00
|
|
|
# Test multiline SQLObserver
|
|
|
|
alert8 = create_alert(
|
|
|
|
dbsession,
|
|
|
|
"""
|
|
|
|
-- comment
|
|
|
|
SELECT
|
|
|
|
1 -- comment
|
|
|
|
FROM test_table
|
|
|
|
WHERE first = 1
|
|
|
|
""",
|
|
|
|
)
|
|
|
|
observe(alert8.id, dbsession)
|
|
|
|
assert alert8.sql_observer[0].observations[-1].value == 1.0
|
|
|
|
assert alert8.sql_observer[0].observations[-1].error_msg is None
|
|
|
|
|
|
|
|
# Test jinja
|
|
|
|
alert9 = create_alert(dbsession, "SELECT {{ 2 }}")
|
|
|
|
observe(alert9.id, dbsession)
|
|
|
|
assert alert9.sql_observer[0].observations[-1].value == 2.0
|
|
|
|
assert alert9.sql_observer[0].observations[-1].error_msg is None
|
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
|
2020-07-28 19:48:42 -04:00
|
|
|
@patch("superset.tasks.schedules.deliver_alert")
|
2020-09-01 16:36:02 -04:00
|
|
|
def test_evaluate_alert(mock_deliver_alert, setup_database):
|
2020-08-10 13:20:43 -04:00
|
|
|
dbsession = setup_database
|
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
# Test error with Observer SQL statement
|
|
|
|
alert1 = create_alert(dbsession, "$%^&")
|
2020-09-10 16:29:57 -04:00
|
|
|
evaluate_alert(alert1.id, alert1.label, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert1.logs[-1].state == AlertState.ERROR
|
2020-08-10 13:20:43 -04:00
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
# Test error with alert lacking observer
|
|
|
|
alert2 = dbsession.query(Alert).filter_by(label="No Observer").one()
|
2020-09-10 16:29:57 -04:00
|
|
|
evaluate_alert(alert2.id, alert2.label, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert2.logs[-1].state == AlertState.ERROR
|
2020-08-10 13:20:43 -04:00
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
# Test pass on alert lacking validator
|
|
|
|
alert3 = create_alert(dbsession, "SELECT 55")
|
2020-09-10 16:29:57 -04:00
|
|
|
evaluate_alert(alert3.id, alert3.label, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert3.logs[-1].state == AlertState.PASS
|
2020-08-06 18:33:48 -04:00
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
# Test triggering successful alert
|
|
|
|
alert4 = create_alert(dbsession, "SELECT 55", "not null", "{}")
|
2020-09-10 16:29:57 -04:00
|
|
|
evaluate_alert(alert4.id, alert4.label, dbsession)
|
2020-08-10 13:20:43 -04:00
|
|
|
assert mock_deliver_alert.call_count == 1
|
2020-09-01 16:36:02 -04:00
|
|
|
assert alert4.logs[-1].state == AlertState.TRIGGER
|
2020-08-06 18:33:48 -04:00
|
|
|
|
2020-07-30 16:27:22 -04:00
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
def test_check_validator():
|
|
|
|
# Test with invalid operator type
|
|
|
|
with pytest.raises(SupersetException):
|
|
|
|
check_validator("greater than", "{}")
|
2020-07-30 16:27:22 -04:00
|
|
|
|
2020-09-01 16:36:02 -04:00
|
|
|
# Test with empty config
|
|
|
|
with pytest.raises(SupersetException):
|
|
|
|
check_validator("operator", "{}")
|
|
|
|
|
|
|
|
# Test with invalid operator
|
|
|
|
with pytest.raises(SupersetException):
|
|
|
|
check_validator("operator", '{"op": "is", "threshold":50.0}')
|
|
|
|
|
|
|
|
# Test with invalid operator
|
|
|
|
with pytest.raises(SupersetException):
|
|
|
|
check_validator("operator", '{"op": "is", "threshold":50.0}')
|
|
|
|
|
|
|
|
# Test with invalid threshold
|
|
|
|
with pytest.raises(SupersetException):
|
|
|
|
check_validator("operator", '{"op": "is", "threshold":"hello"}')
|
|
|
|
|
|
|
|
# Test with float threshold and no errors
|
|
|
|
assert check_validator("operator", '{"op": ">=", "threshold": 50.0}') is None
|
|
|
|
|
|
|
|
# Test with int threshold and no errors
|
|
|
|
assert check_validator("operator", '{"op": "==", "threshold": 50}') is None
|
|
|
|
|
|
|
|
|
|
|
|
def test_not_null_validator(setup_database):
|
|
|
|
dbsession = setup_database
|
|
|
|
|
|
|
|
# Test passing SQLObserver with 'null' SQL result
|
|
|
|
alert1 = create_alert(dbsession, "SELECT 0")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert1.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert not_null_validator(alert1.sql_observer[0], "{}") is False
|
|
|
|
|
|
|
|
# Test passing SQLObserver with empty SQL result
|
|
|
|
alert2 = create_alert(dbsession, "SELECT first FROM test_table WHERE first = -1")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert2.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert not_null_validator(alert2.sql_observer[0], "{}") is False
|
|
|
|
|
|
|
|
# Test triggering alert with non-null SQL result
|
|
|
|
alert3 = create_alert(dbsession, "SELECT 55")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert3.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert not_null_validator(alert3.sql_observer[0], "{}") is True
|
|
|
|
|
|
|
|
|
|
|
|
def test_operator_validator(setup_database):
|
2020-08-10 13:20:43 -04:00
|
|
|
dbsession = setup_database
|
2020-09-01 16:36:02 -04:00
|
|
|
|
|
|
|
# Test passing SQLObserver with empty SQL result
|
|
|
|
alert1 = create_alert(dbsession, "SELECT first FROM test_table WHERE first = -1")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert1.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert (
|
|
|
|
operator_validator(alert1.sql_observer[0], '{"op": ">=", "threshold": 60}')
|
|
|
|
is False
|
2020-08-06 18:33:48 -04:00
|
|
|
)
|
2020-09-16 14:09:11 -04:00
|
|
|
# ensure that 0 threshold works
|
|
|
|
assert (
|
|
|
|
operator_validator(alert1.sql_observer[0], '{"op": ">=", "threshold": 0}')
|
|
|
|
is False
|
|
|
|
)
|
2020-09-01 16:36:02 -04:00
|
|
|
|
|
|
|
# Test passing SQLObserver with result that doesn't pass a greater than threshold
|
|
|
|
alert2 = create_alert(dbsession, "SELECT 55")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert2.id, dbsession)
|
2020-09-01 16:36:02 -04:00
|
|
|
assert (
|
|
|
|
operator_validator(alert2.sql_observer[0], '{"op": ">=", "threshold": 60}')
|
|
|
|
is False
|
|
|
|
)
|
|
|
|
|
|
|
|
# Test passing SQLObserver with result that passes a greater than threshold
|
|
|
|
assert (
|
|
|
|
operator_validator(alert2.sql_observer[0], '{"op": ">=", "threshold": 40}')
|
|
|
|
is True
|
|
|
|
)
|
|
|
|
|
|
|
|
# Test passing SQLObserver with result that doesn't pass a less than threshold
|
|
|
|
assert (
|
|
|
|
operator_validator(alert2.sql_observer[0], '{"op": "<=", "threshold": 40}')
|
|
|
|
is False
|
|
|
|
)
|
|
|
|
|
|
|
|
# Test passing SQLObserver with result that passes threshold
|
|
|
|
assert (
|
|
|
|
operator_validator(alert2.sql_observer[0], '{"op": "<=", "threshold": 60}')
|
|
|
|
is True
|
|
|
|
)
|
|
|
|
|
|
|
|
# Test passing SQLObserver with result that doesn't equal threshold
|
|
|
|
assert (
|
|
|
|
operator_validator(alert2.sql_observer[0], '{"op": "==", "threshold": 60}')
|
|
|
|
is False
|
|
|
|
)
|
|
|
|
|
|
|
|
# Test passing SQLObserver with result that equals threshold
|
|
|
|
assert (
|
|
|
|
operator_validator(alert2.sql_observer[0], '{"op": "==", "threshold": 55}')
|
|
|
|
is True
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def test_validate_observations(setup_database):
|
|
|
|
dbsession = setup_database
|
|
|
|
|
|
|
|
# Test False on alert with no validator
|
|
|
|
alert1 = create_alert(dbsession, "SELECT 55")
|
2020-09-10 16:29:57 -04:00
|
|
|
assert validate_observations(alert1.id, alert1.label, dbsession) is False
|
2020-09-01 16:36:02 -04:00
|
|
|
|
|
|
|
# Test False on alert with no observations
|
|
|
|
alert2 = create_alert(dbsession, "SELECT 55", "not null", "{}")
|
2020-09-10 16:29:57 -04:00
|
|
|
assert validate_observations(alert2.id, alert2.label, dbsession) is False
|
2020-09-01 16:36:02 -04:00
|
|
|
|
|
|
|
# Test False on alert that shouldnt be triggered
|
|
|
|
alert3 = create_alert(dbsession, "SELECT 0", "not null", "{}")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert3.id, dbsession)
|
|
|
|
assert validate_observations(alert3.id, alert3.label, dbsession) is False
|
2020-09-01 16:36:02 -04:00
|
|
|
|
|
|
|
# Test True on alert that should be triggered
|
|
|
|
alert4 = create_alert(
|
|
|
|
dbsession, "SELECT 55", "operator", '{"op": "<=", "threshold": 60}'
|
|
|
|
)
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert4.id, dbsession)
|
|
|
|
assert validate_observations(alert4.id, alert4.label, dbsession) is True
|
2020-08-11 14:15:31 -04:00
|
|
|
|
|
|
|
|
|
|
|
@patch("superset.tasks.slack_util.WebClient.files_upload")
|
|
|
|
@patch("superset.tasks.schedules.send_email_smtp")
|
|
|
|
@patch("superset.tasks.schedules._get_url_path")
|
|
|
|
@patch("superset.utils.screenshots.ChartScreenshot.compute_and_cache")
|
|
|
|
def test_deliver_alert_screenshot(
|
|
|
|
screenshot_mock, url_mock, email_mock, file_upload_mock, setup_database
|
|
|
|
):
|
|
|
|
dbsession = setup_database
|
2020-09-03 11:15:05 -04:00
|
|
|
alert = create_alert(dbsession, "SELECT 55", "not null", "{}")
|
2020-09-10 16:29:57 -04:00
|
|
|
observe(alert.id, dbsession)
|
2020-08-11 14:15:31 -04:00
|
|
|
|
|
|
|
screenshot = read_fixture("sample.png")
|
|
|
|
screenshot_mock.return_value = screenshot
|
|
|
|
|
|
|
|
# TODO: fix AlertModelView.show url call from test
|
|
|
|
url_mock.side_effect = [
|
|
|
|
f"http://0.0.0.0:8080/alert/show/{alert.id}",
|
|
|
|
f"http://0.0.0.0:8080/superset/slice/{alert.slice_id}/",
|
|
|
|
]
|
|
|
|
|
2020-09-10 16:29:57 -04:00
|
|
|
deliver_alert(alert.id, dbsession)
|
2020-08-11 14:15:31 -04:00
|
|
|
assert email_mock.call_args[1]["images"]["screenshot"] == screenshot
|
|
|
|
assert file_upload_mock.call_args[1] == {
|
|
|
|
"channels": alert.slack_channel,
|
|
|
|
"file": screenshot,
|
|
|
|
"initial_comment": f"\n*Triggered Alert: {alert.label} :redalert:*\n"
|
2020-09-03 11:15:05 -04:00
|
|
|
f"*Query*:```{alert.sql_observer[0].sql}```\n"
|
|
|
|
f"*Result*: {alert.observations[-1].value}\n"
|
2020-09-16 14:09:11 -04:00
|
|
|
f"*Reason*: {alert.observations[-1].value} {alert.validators[0].pretty_config}\n"
|
2020-09-03 11:15:05 -04:00
|
|
|
f"<http://0.0.0.0:8080/alert/show/{alert.id}"
|
2020-08-11 14:15:31 -04:00
|
|
|
f"|View Alert Details>\n<http://0.0.0.0:8080/superset/slice/{alert.slice_id}/"
|
|
|
|
"|*Explore in Superset*>",
|
|
|
|
"title": f"[Alert] {alert.label}",
|
|
|
|
}
|