feat: add alerts & reports to docker compose (#14267)

* feat: add alerts & reports to docker compose

* change to firefox

* add missing package

Dockerfile

@@ -123,10 +123,27 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 # Dev image...
 ######################################################################
 FROM lean AS dev
+ARG GECKODRIVER_VERSION=v0.28.0
+ARG FIREFOX_VERSION=88.0
 
 COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/
 
 USER root
+
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends libnss3 libdbus-glib-1-2 libgtk-3-0 libx11-xcb1
+
+# Install GeckoDriver WebDriver
+RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O /tmp/geckodriver.tar.gz && \
+    tar xvfz /tmp/geckodriver.tar.gz -C /tmp && \
+    mv /tmp/geckodriver /usr/local/bin/geckodriver && \
+    rm /tmp/geckodriver.tar.gz
+
+# Install Firefox
+RUN wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O /opt/firefox.tar.bz2 && \
+    tar xvf /opt/firefox.tar.bz2 -C /opt && \
+    ln -s /opt/firefox/firefox /usr/local/bin/firefox
+
 # Cache everything for dev purposes...
 RUN cd /app \
     && pip install --no-cache -r requirements/docker.txt \

docker/pythonpath_dev/superset_config.py

@@ -20,11 +20,12 @@
 # development environments. Also note that superset_config_docker.py is imported
 # as a final step as a means to override "defaults" configured here
 #
-
 import logging
 import os
+from datetime import timedelta
 
 from cachelib.file import FileSystemCache
+from celery.schedules import crontab
 
 logger = logging.getLogger()
 
@@ -70,13 +71,31 @@ RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
 
 class CeleryConfig(object):
     BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
-    CELERY_IMPORTS = ("superset.sql_lab",)
+    CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks")
     CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
-    CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}}
-    CELERY_TASK_PROTOCOL = 1
+    CELERYD_LOG_LEVEL = "DEBUG"
+    CELERYD_PREFETCH_MULTIPLIER = 1
+    CELERY_ACKS_LATE = False
+    CELERYBEAT_SCHEDULE = {
+        "reports.scheduler": {
+            "task": "reports.scheduler",
+            "schedule": crontab(minute="*", hour="*"),
+        },
+        "reports.prune_log": {
+            "task": "reports.prune_log",
+            "schedule": crontab(minute=10, hour=0),
+        },
+    }
 
 
 CELERY_CONFIG = CeleryConfig
+
+FEATURE_FLAGS = {"ALERT_REPORTS": True}
+ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
+WEBDRIVER_BASEURL = "http://superset:8088/"
+# The base URL for the email report hyperlinks.
+WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
+
 SQLLAB_CTAS_NO_LIMIT = True
 
 #

superset/config.py

@@ -956,6 +956,9 @@ ALERT_REPORTS_WORKING_TIME_OUT_LAG = 10
 # if ALERT_REPORTS_WORKING_TIME_OUT_KILL is True, set a celery hard timeout
 # Equal to working timeout + ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG
 ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG = 1
+# If set to true no notification is sent, the worker will just log a message.
+# Useful for debugging
+ALERT_REPORTS_NOTIFICATION_DRY_RUN = False
 
 # A custom prefix to use on all Alerts & Reports emails
 EMAIL_REPORTS_SUBJECT_PREFIX = "[Report] "

superset/reports/commands/execute.py

@@ -270,9 +270,10 @@ class BaseReportState:
             csv=csv_data,
         )
 
-    @staticmethod
     def _send(
-        notification_content: NotificationContent, recipients: List[ReportRecipients]
+        self,
+        notification_content: NotificationContent,
+        recipients: List[ReportRecipients],
     ) -> None:
         """
         Sends a notification to all recipients
@@ -283,7 +284,14 @@ class BaseReportState:
         for recipient in recipients:
             notification = create_notification(recipient, notification_content)
             try:
-                notification.send()
+                if app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"]:
+                    logger.info(
+                        "Would send notification for alert %s, to %s",
+                        self._report_schedule.name,
+                        recipient.recipient_config_json,
+                    )
+                else:
+                    notification.send()
             except NotificationError as ex:
                 # collect notification errors but keep processing them
                 notification_errors.append(str(ex))

tests/reports/commands_tests.py

@@ -568,7 +568,7 @@ def test_email_chart_report_schedule(
     screenshot_mock, email_mock, create_report_email_chart,
 ):
     """
-    ExecuteReport Command: Test chart email report schedule with
+    ExecuteReport Command: Test chart email report schedule with screenshot
     """
     # setup screenshot mock
     screenshot_mock.return_value = SCREENSHOT_FILE
@@ -596,6 +596,29 @@ def test_email_chart_report_schedule(
         assert_log(ReportState.SUCCESS)
 
 
+@pytest.mark.usefixtures(
+    "load_birth_names_dashboard_with_slices", "create_report_email_chart"
+)
+@patch("superset.reports.notifications.email.send_email_smtp")
+@patch("superset.utils.screenshots.ChartScreenshot.get_screenshot")
+def test_email_chart_report_dry_run(
+    screenshot_mock, email_mock, create_report_email_chart,
+):
+    """
+    ExecuteReport Command: Test chart email report schedule dry run
+    """
+    # setup screenshot mock
+    screenshot_mock.return_value = SCREENSHOT_FILE
+    app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"] = True
+    with freeze_time("2020-01-01T00:00:00Z"):
+        AsyncExecuteReportScheduleCommand(
+            TEST_ID, create_report_email_chart.id, datetime.utcnow()
+        ).run()
+
+        email_mock.assert_not_called()
+    app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"] = False
+
+
 @pytest.mark.usefixtures(
     "load_birth_names_dashboard_with_slices", "create_report_email_chart_with_csv"
 )