mirror of https://github.com/apache/superset.git
feat: new report schedule models (#11550)
* feat: new report schedule models * lint and unique constraint * support sqlite * fix sqlite * add audit mixin and minor fixes * fix FK's * address comments * lint
This commit is contained in:
parent
6d5d92a6fe
commit
bd79bd2a54
|
@ -0,0 +1,133 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""add report schedules
|
||||
|
||||
Revision ID: 49b5a32daba5
|
||||
Revises: 96e99fb176a0
|
||||
Create Date: 2020-11-04 11:06:59.249758
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "49b5a32daba5"
|
||||
down_revision = "96e99fb176a0"
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"report_schedule",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("type", sa.String(length=50), nullable=False),
|
||||
sa.Column("name", sa.String(length=150), nullable=False, unique=True),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("context_markdown", sa.Text(), nullable=True),
|
||||
sa.Column("active", sa.Boolean(), default=True, nullable=True),
|
||||
sa.Column("crontab", sa.String(length=50), nullable=False),
|
||||
sa.Column("sql", sa.Text(), nullable=True),
|
||||
sa.Column("chart_id", sa.Integer(), nullable=True),
|
||||
sa.Column("dashboard_id", sa.Integer(), nullable=True),
|
||||
sa.Column("database_id", sa.Integer(), nullable=True),
|
||||
sa.Column("last_eval_dttm", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_state", sa.String(length=50), nullable=True),
|
||||
sa.Column("last_value", sa.Float(), nullable=True),
|
||||
sa.Column("last_value_row_json", sa.Text(), nullable=True),
|
||||
sa.Column("validator_type", sa.String(length=100), nullable=True),
|
||||
sa.Column("validator_config_json", sa.Text(), default="{}", nullable=True),
|
||||
sa.Column("log_retention", sa.Integer(), nullable=True, default=90),
|
||||
sa.Column("grace_period", sa.Integer(), nullable=True, default=60 * 60 * 4),
|
||||
# Audit Mixin
|
||||
sa.Column("created_on", sa.DateTime(), nullable=True),
|
||||
sa.Column("changed_on", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_by_fk", sa.Integer(), nullable=True),
|
||||
sa.Column("changed_by_fk", sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(["chart_id"], ["slices.id"]),
|
||||
sa.ForeignKeyConstraint(["dashboard_id"], ["dashboards.id"]),
|
||||
sa.ForeignKeyConstraint(["database_id"], ["dbs.id"]),
|
||||
sa.ForeignKeyConstraint(["changed_by_fk"], ["ab_user.id"]),
|
||||
sa.ForeignKeyConstraint(["created_by_fk"], ["ab_user.id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
try:
|
||||
op.create_unique_constraint(
|
||||
"uq_report_schedule_name", "report_schedule", ["name"]
|
||||
)
|
||||
except Exception:
|
||||
# Expected to fail on SQLite
|
||||
pass
|
||||
op.create_index(
|
||||
op.f("ix_report_schedule_active"), "report_schedule", ["active"], unique=False
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"report_execution_log",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("scheduled_dttm", sa.DateTime(), nullable=False),
|
||||
sa.Column("start_dttm", sa.DateTime(), nullable=True),
|
||||
sa.Column("end_dttm", sa.DateTime(), nullable=True),
|
||||
sa.Column("value", sa.Float(), nullable=True),
|
||||
sa.Column("value_row_json", sa.Text(), nullable=True),
|
||||
sa.Column("state", sa.String(length=50), nullable=False),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column("report_schedule_id", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"report_recipient",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("type", sa.String(length=50), nullable=False),
|
||||
sa.Column("recipient_config_json", sa.Text(), default="{}", nullable=True),
|
||||
sa.Column("report_schedule_id", sa.Integer(), nullable=False),
|
||||
# Audit Mixin
|
||||
sa.Column("created_on", sa.DateTime(), nullable=True),
|
||||
sa.Column("changed_on", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_by_fk", sa.Integer(), nullable=True),
|
||||
sa.Column("changed_by_fk", sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"]),
|
||||
sa.ForeignKeyConstraint(["changed_by_fk"], ["ab_user.id"]),
|
||||
sa.ForeignKeyConstraint(["created_by_fk"], ["ab_user.id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"report_schedule_user",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||
sa.Column("report_schedule_id", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"],),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["ab_user.id"],),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index(op.f("ix_report_schedule_active"), table_name="report_schedule")
|
||||
try:
|
||||
op.drop_constraint("uq_report_schedule_name", "report_schedule", type_="unique")
|
||||
except Exception:
|
||||
# Expected to fail on SQLite
|
||||
pass
|
||||
|
||||
op.drop_table("report_execution_log")
|
||||
op.drop_table("report_recipient")
|
||||
op.drop_table("report_schedule_user")
|
||||
op.drop_table("report_schedule")
|
|
@ -0,0 +1,177 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# pylint: disable=line-too-long,unused-argument,ungrouped-imports
|
||||
"""A collection of ORM sqlalchemy models for Superset"""
|
||||
import enum
|
||||
|
||||
from flask_appbuilder import Model
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
Column,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Table,
|
||||
Text,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.schema import UniqueConstraint
|
||||
|
||||
from superset.extensions import security_manager
|
||||
from superset.models.core import Database
|
||||
from superset.models.dashboard import Dashboard
|
||||
from superset.models.helpers import AuditMixinNullable
|
||||
from superset.models.slice import Slice
|
||||
|
||||
metadata = Model.metadata # pylint: disable=no-member
|
||||
|
||||
|
||||
class ReportScheduleType(str, enum.Enum):
|
||||
ALERT = "Alert"
|
||||
REPORT = "Report"
|
||||
|
||||
|
||||
class ReportScheduleValidatorType(str, enum.Enum):
|
||||
""" Validator types for alerts """
|
||||
|
||||
not_null = "not null"
|
||||
operator = "operator"
|
||||
|
||||
|
||||
class ReportRecipientType(str, enum.Enum):
|
||||
EMAIL = "Email"
|
||||
SLACK = "Slack"
|
||||
|
||||
|
||||
class ReportLogState(str, enum.Enum):
|
||||
SUCCESS = "Success"
|
||||
ERROR = "Error"
|
||||
|
||||
|
||||
class ReportEmailFormat(str, enum.Enum):
|
||||
VISUALIZATION = "Visualization"
|
||||
DATA = "Raw data"
|
||||
|
||||
|
||||
report_schedule_user = Table(
|
||||
"report_schedule_user",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("user_id", Integer, ForeignKey("ab_user.id"), nullable=False),
|
||||
Column(
|
||||
"report_schedule_id", Integer, ForeignKey("report_schedule.id"), nullable=False
|
||||
),
|
||||
UniqueConstraint("user_id", "report_schedule_id"),
|
||||
)
|
||||
|
||||
|
||||
class ReportSchedule(Model, AuditMixinNullable):
|
||||
|
||||
"""
|
||||
Report Schedules, supports alerts and reports
|
||||
"""
|
||||
|
||||
__tablename__ = "report_schedule"
|
||||
id = Column(Integer, primary_key=True)
|
||||
type = Column(String(50), nullable=False)
|
||||
name = Column(String(150), nullable=False, unique=True)
|
||||
description = Column(Text)
|
||||
context_markdown = Column(Text)
|
||||
active = Column(Boolean, default=True, index=True)
|
||||
crontab = Column(String(50), nullable=False)
|
||||
sql = Column(Text())
|
||||
# (Alerts/Reports) M-O to chart
|
||||
chart_id = Column(Integer, ForeignKey("slices.id"), nullable=True)
|
||||
chart = relationship(Slice, backref="report_schedules", foreign_keys=[chart_id])
|
||||
# (Alerts/Reports) M-O to dashboard
|
||||
dashboard_id = Column(Integer, ForeignKey("dashboards.id"), nullable=True)
|
||||
dashboard = relationship(
|
||||
Dashboard, backref="report_schedules", foreign_keys=[dashboard_id]
|
||||
)
|
||||
# (Alerts) M-O to database
|
||||
database_id = Column(Integer, ForeignKey("dbs.id"), nullable=True)
|
||||
database = relationship(Database, foreign_keys=[database_id])
|
||||
owners = relationship(security_manager.user_model, secondary=report_schedule_user)
|
||||
|
||||
# (Alerts) Stamped last observations
|
||||
last_eval_dttm = Column(DateTime)
|
||||
last_state = Column(String(50))
|
||||
last_value = Column(Float)
|
||||
last_value_row_json = Column(Text)
|
||||
|
||||
# (Alerts) Observed value validation related columns
|
||||
validator_type = Column(String(100))
|
||||
validator_config_json = Column(Text, default="{}")
|
||||
|
||||
# Log retention
|
||||
log_retention = Column(Integer, default=90)
|
||||
grace_period = Column(Integer, default=60 * 60 * 4)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return str(self.name)
|
||||
|
||||
|
||||
class ReportRecipients(
|
||||
Model, AuditMixinNullable
|
||||
): # pylint: disable=too-few-public-methods
|
||||
|
||||
"""
|
||||
Report Recipients, meant to support multiple notification types, eg: Slack, email
|
||||
"""
|
||||
|
||||
__tablename__ = "report_recipient"
|
||||
id = Column(Integer, primary_key=True)
|
||||
type = Column(String(50), nullable=False)
|
||||
recipient_config_json = Column(Text, default="{}")
|
||||
report_schedule_id = Column(
|
||||
Integer, ForeignKey("report_schedule.id"), nullable=False
|
||||
)
|
||||
report_schedule = relationship(
|
||||
ReportSchedule, backref="recipients", foreign_keys=[report_schedule_id]
|
||||
)
|
||||
|
||||
|
||||
class ReportExecutionLog(Model): # pylint: disable=too-few-public-methods
|
||||
|
||||
"""
|
||||
Report Execution Log, hold the result of the report execution with timestamps,
|
||||
last observation and possible error messages
|
||||
"""
|
||||
|
||||
__tablename__ = "report_execution_log"
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Timestamps
|
||||
scheduled_dttm = Column(DateTime, nullable=False)
|
||||
start_dttm = Column(DateTime)
|
||||
end_dttm = Column(DateTime)
|
||||
|
||||
# (Alerts) Observed values
|
||||
value = Column(Float)
|
||||
value_row_json = Column(Text)
|
||||
|
||||
state = Column(String(50), nullable=False)
|
||||
error_message = Column(Text)
|
||||
|
||||
report_schedule_id = Column(
|
||||
Integer, ForeignKey("report_schedule.id"), nullable=False
|
||||
)
|
||||
report_schedule = relationship(
|
||||
ReportSchedule, backref="logs", foreign_keys=[report_schedule_id]
|
||||
)
|
Loading…
Reference in New Issue