From bd79bd2a54e5b8dc1895e0e56c130ba57f0bb4c8 Mon Sep 17 00:00:00 2001 From: Daniel Vaz Gaspar Date: Fri, 6 Nov 2020 09:33:32 +0000 Subject: [PATCH] feat: new report schedule models (#11550) * feat: new report schedule models * lint and unique constraint * support sqlite * fix sqlite * add audit mixin and minor fixes * fix FK's * address comments * lint --- .../49b5a32daba5_add_report_schedules.py | 133 +++++++++++++ superset/models/reports.py | 177 ++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 superset/migrations/versions/49b5a32daba5_add_report_schedules.py create mode 100644 superset/models/reports.py diff --git a/superset/migrations/versions/49b5a32daba5_add_report_schedules.py b/superset/migrations/versions/49b5a32daba5_add_report_schedules.py new file mode 100644 index 0000000000..f20745471e --- /dev/null +++ b/superset/migrations/versions/49b5a32daba5_add_report_schedules.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""add report schedules + +Revision ID: 49b5a32daba5 +Revises: 96e99fb176a0 +Create Date: 2020-11-04 11:06:59.249758 + +""" + +# revision identifiers, used by Alembic. +revision = "49b5a32daba5" +down_revision = "96e99fb176a0" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.exc import OperationalError + + +def upgrade(): + op.create_table( + "report_schedule", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("type", sa.String(length=50), nullable=False), + sa.Column("name", sa.String(length=150), nullable=False, unique=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("context_markdown", sa.Text(), nullable=True), + sa.Column("active", sa.Boolean(), default=True, nullable=True), + sa.Column("crontab", sa.String(length=50), nullable=False), + sa.Column("sql", sa.Text(), nullable=True), + sa.Column("chart_id", sa.Integer(), nullable=True), + sa.Column("dashboard_id", sa.Integer(), nullable=True), + sa.Column("database_id", sa.Integer(), nullable=True), + sa.Column("last_eval_dttm", sa.DateTime(), nullable=True), + sa.Column("last_state", sa.String(length=50), nullable=True), + sa.Column("last_value", sa.Float(), nullable=True), + sa.Column("last_value_row_json", sa.Text(), nullable=True), + sa.Column("validator_type", sa.String(length=100), nullable=True), + sa.Column("validator_config_json", sa.Text(), default="{}", nullable=True), + sa.Column("log_retention", sa.Integer(), nullable=True, default=90), + sa.Column("grace_period", sa.Integer(), nullable=True, default=60 * 60 * 4), + # Audit Mixin + sa.Column("created_on", sa.DateTime(), nullable=True), + sa.Column("changed_on", sa.DateTime(), nullable=True), + sa.Column("created_by_fk", sa.Integer(), nullable=True), + sa.Column("changed_by_fk", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(["chart_id"], ["slices.id"]), + sa.ForeignKeyConstraint(["dashboard_id"], ["dashboards.id"]), + sa.ForeignKeyConstraint(["database_id"], ["dbs.id"]), + sa.ForeignKeyConstraint(["changed_by_fk"], ["ab_user.id"]), + sa.ForeignKeyConstraint(["created_by_fk"], ["ab_user.id"]), + sa.PrimaryKeyConstraint("id"), + ) + try: + op.create_unique_constraint( + "uq_report_schedule_name", "report_schedule", ["name"] + ) + except Exception: + # Expected to fail on SQLite + pass + op.create_index( + op.f("ix_report_schedule_active"), "report_schedule", ["active"], unique=False + ) + + op.create_table( + "report_execution_log", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("scheduled_dttm", sa.DateTime(), nullable=False), + sa.Column("start_dttm", sa.DateTime(), nullable=True), + sa.Column("end_dttm", sa.DateTime(), nullable=True), + sa.Column("value", sa.Float(), nullable=True), + sa.Column("value_row_json", sa.Text(), nullable=True), + sa.Column("state", sa.String(length=50), nullable=False), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("report_schedule_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"]), + sa.PrimaryKeyConstraint("id"), + ) + + op.create_table( + "report_recipient", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("type", sa.String(length=50), nullable=False), + sa.Column("recipient_config_json", sa.Text(), default="{}", nullable=True), + sa.Column("report_schedule_id", sa.Integer(), nullable=False), + # Audit Mixin + sa.Column("created_on", sa.DateTime(), nullable=True), + sa.Column("changed_on", sa.DateTime(), nullable=True), + sa.Column("created_by_fk", sa.Integer(), nullable=True), + sa.Column("changed_by_fk", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"]), + sa.ForeignKeyConstraint(["changed_by_fk"], ["ab_user.id"]), + sa.ForeignKeyConstraint(["created_by_fk"], ["ab_user.id"]), + sa.PrimaryKeyConstraint("id"), + ) + + op.create_table( + "report_schedule_user", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("report_schedule_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"],), + sa.ForeignKeyConstraint(["user_id"], ["ab_user.id"],), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_index(op.f("ix_report_schedule_active"), table_name="report_schedule") + try: + op.drop_constraint("uq_report_schedule_name", "report_schedule", type_="unique") + except Exception: + # Expected to fail on SQLite + pass + + op.drop_table("report_execution_log") + op.drop_table("report_recipient") + op.drop_table("report_schedule_user") + op.drop_table("report_schedule") diff --git a/superset/models/reports.py b/superset/models/reports.py new file mode 100644 index 0000000000..c510cc6069 --- /dev/null +++ b/superset/models/reports.py @@ -0,0 +1,177 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=line-too-long,unused-argument,ungrouped-imports +"""A collection of ORM sqlalchemy models for Superset""" +import enum + +from flask_appbuilder import Model +from sqlalchemy import ( + Boolean, + Column, + DateTime, + Float, + ForeignKey, + Integer, + String, + Table, + Text, +) +from sqlalchemy.orm import relationship +from sqlalchemy.schema import UniqueConstraint + +from superset.extensions import security_manager +from superset.models.core import Database +from superset.models.dashboard import Dashboard +from superset.models.helpers import AuditMixinNullable +from superset.models.slice import Slice + +metadata = Model.metadata # pylint: disable=no-member + + +class ReportScheduleType(str, enum.Enum): + ALERT = "Alert" + REPORT = "Report" + + +class ReportScheduleValidatorType(str, enum.Enum): + """ Validator types for alerts """ + + not_null = "not null" + operator = "operator" + + +class ReportRecipientType(str, enum.Enum): + EMAIL = "Email" + SLACK = "Slack" + + +class ReportLogState(str, enum.Enum): + SUCCESS = "Success" + ERROR = "Error" + + +class ReportEmailFormat(str, enum.Enum): + VISUALIZATION = "Visualization" + DATA = "Raw data" + + +report_schedule_user = Table( + "report_schedule_user", + metadata, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("ab_user.id"), nullable=False), + Column( + "report_schedule_id", Integer, ForeignKey("report_schedule.id"), nullable=False + ), + UniqueConstraint("user_id", "report_schedule_id"), +) + + +class ReportSchedule(Model, AuditMixinNullable): + + """ + Report Schedules, supports alerts and reports + """ + + __tablename__ = "report_schedule" + id = Column(Integer, primary_key=True) + type = Column(String(50), nullable=False) + name = Column(String(150), nullable=False, unique=True) + description = Column(Text) + context_markdown = Column(Text) + active = Column(Boolean, default=True, index=True) + crontab = Column(String(50), nullable=False) + sql = Column(Text()) + # (Alerts/Reports) M-O to chart + chart_id = Column(Integer, ForeignKey("slices.id"), nullable=True) + chart = relationship(Slice, backref="report_schedules", foreign_keys=[chart_id]) + # (Alerts/Reports) M-O to dashboard + dashboard_id = Column(Integer, ForeignKey("dashboards.id"), nullable=True) + dashboard = relationship( + Dashboard, backref="report_schedules", foreign_keys=[dashboard_id] + ) + # (Alerts) M-O to database + database_id = Column(Integer, ForeignKey("dbs.id"), nullable=True) + database = relationship(Database, foreign_keys=[database_id]) + owners = relationship(security_manager.user_model, secondary=report_schedule_user) + + # (Alerts) Stamped last observations + last_eval_dttm = Column(DateTime) + last_state = Column(String(50)) + last_value = Column(Float) + last_value_row_json = Column(Text) + + # (Alerts) Observed value validation related columns + validator_type = Column(String(100)) + validator_config_json = Column(Text, default="{}") + + # Log retention + log_retention = Column(Integer, default=90) + grace_period = Column(Integer, default=60 * 60 * 4) + + def __repr__(self) -> str: + return str(self.name) + + +class ReportRecipients( + Model, AuditMixinNullable +): # pylint: disable=too-few-public-methods + + """ + Report Recipients, meant to support multiple notification types, eg: Slack, email + """ + + __tablename__ = "report_recipient" + id = Column(Integer, primary_key=True) + type = Column(String(50), nullable=False) + recipient_config_json = Column(Text, default="{}") + report_schedule_id = Column( + Integer, ForeignKey("report_schedule.id"), nullable=False + ) + report_schedule = relationship( + ReportSchedule, backref="recipients", foreign_keys=[report_schedule_id] + ) + + +class ReportExecutionLog(Model): # pylint: disable=too-few-public-methods + + """ + Report Execution Log, hold the result of the report execution with timestamps, + last observation and possible error messages + """ + + __tablename__ = "report_execution_log" + id = Column(Integer, primary_key=True) + + # Timestamps + scheduled_dttm = Column(DateTime, nullable=False) + start_dttm = Column(DateTime) + end_dttm = Column(DateTime) + + # (Alerts) Observed values + value = Column(Float) + value_row_json = Column(Text) + + state = Column(String(50), nullable=False) + error_message = Column(Text) + + report_schedule_id = Column( + Integer, ForeignKey("report_schedule.id"), nullable=False + ) + report_schedule = relationship( + ReportSchedule, backref="logs", foreign_keys=[report_schedule_id] + )