mirror of
https://github.com/apache/superset.git
synced 2024-09-12 00:29:39 -04:00
feat: new report schedule models (#11550)
* feat: new report schedule models * lint and unique constraint * support sqlite * fix sqlite * add audit mixin and minor fixes * fix FK's * address comments * lint
This commit is contained in:
parent
6d5d92a6fe
commit
bd79bd2a54
@ -0,0 +1,133 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
"""add report schedules
|
||||||
|
|
||||||
|
Revision ID: 49b5a32daba5
|
||||||
|
Revises: 96e99fb176a0
|
||||||
|
Create Date: 2020-11-04 11:06:59.249758
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = "49b5a32daba5"
|
||||||
|
down_revision = "96e99fb176a0"
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.exc import OperationalError
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.create_table(
|
||||||
|
"report_schedule",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("type", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=150), nullable=False, unique=True),
|
||||||
|
sa.Column("description", sa.Text(), nullable=True),
|
||||||
|
sa.Column("context_markdown", sa.Text(), nullable=True),
|
||||||
|
sa.Column("active", sa.Boolean(), default=True, nullable=True),
|
||||||
|
sa.Column("crontab", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("sql", sa.Text(), nullable=True),
|
||||||
|
sa.Column("chart_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("dashboard_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("database_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("last_eval_dttm", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("last_state", sa.String(length=50), nullable=True),
|
||||||
|
sa.Column("last_value", sa.Float(), nullable=True),
|
||||||
|
sa.Column("last_value_row_json", sa.Text(), nullable=True),
|
||||||
|
sa.Column("validator_type", sa.String(length=100), nullable=True),
|
||||||
|
sa.Column("validator_config_json", sa.Text(), default="{}", nullable=True),
|
||||||
|
sa.Column("log_retention", sa.Integer(), nullable=True, default=90),
|
||||||
|
sa.Column("grace_period", sa.Integer(), nullable=True, default=60 * 60 * 4),
|
||||||
|
# Audit Mixin
|
||||||
|
sa.Column("created_on", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("changed_on", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("created_by_fk", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("changed_by_fk", sa.Integer(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(["chart_id"], ["slices.id"]),
|
||||||
|
sa.ForeignKeyConstraint(["dashboard_id"], ["dashboards.id"]),
|
||||||
|
sa.ForeignKeyConstraint(["database_id"], ["dbs.id"]),
|
||||||
|
sa.ForeignKeyConstraint(["changed_by_fk"], ["ab_user.id"]),
|
||||||
|
sa.ForeignKeyConstraint(["created_by_fk"], ["ab_user.id"]),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_report_schedule_name", "report_schedule", ["name"]
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Expected to fail on SQLite
|
||||||
|
pass
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_report_schedule_active"), "report_schedule", ["active"], unique=False
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"report_execution_log",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("scheduled_dttm", sa.DateTime(), nullable=False),
|
||||||
|
sa.Column("start_dttm", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("end_dttm", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("value", sa.Float(), nullable=True),
|
||||||
|
sa.Column("value_row_json", sa.Text(), nullable=True),
|
||||||
|
sa.Column("state", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column("report_schedule_id", sa.Integer(), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"]),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"report_recipient",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("type", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("recipient_config_json", sa.Text(), default="{}", nullable=True),
|
||||||
|
sa.Column("report_schedule_id", sa.Integer(), nullable=False),
|
||||||
|
# Audit Mixin
|
||||||
|
sa.Column("created_on", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("changed_on", sa.DateTime(), nullable=True),
|
||||||
|
sa.Column("created_by_fk", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("changed_by_fk", sa.Integer(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"]),
|
||||||
|
sa.ForeignKeyConstraint(["changed_by_fk"], ["ab_user.id"]),
|
||||||
|
sa.ForeignKeyConstraint(["created_by_fk"], ["ab_user.id"]),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"report_schedule_user",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("report_schedule_id", sa.Integer(), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(["report_schedule_id"], ["report_schedule.id"],),
|
||||||
|
sa.ForeignKeyConstraint(["user_id"], ["ab_user.id"],),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_index(op.f("ix_report_schedule_active"), table_name="report_schedule")
|
||||||
|
try:
|
||||||
|
op.drop_constraint("uq_report_schedule_name", "report_schedule", type_="unique")
|
||||||
|
except Exception:
|
||||||
|
# Expected to fail on SQLite
|
||||||
|
pass
|
||||||
|
|
||||||
|
op.drop_table("report_execution_log")
|
||||||
|
op.drop_table("report_recipient")
|
||||||
|
op.drop_table("report_schedule_user")
|
||||||
|
op.drop_table("report_schedule")
|
177
superset/models/reports.py
Normal file
177
superset/models/reports.py
Normal file
@ -0,0 +1,177 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
# pylint: disable=line-too-long,unused-argument,ungrouped-imports
|
||||||
|
"""A collection of ORM sqlalchemy models for Superset"""
|
||||||
|
import enum
|
||||||
|
|
||||||
|
from flask_appbuilder import Model
|
||||||
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
Column,
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Table,
|
||||||
|
Text,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
from sqlalchemy.schema import UniqueConstraint
|
||||||
|
|
||||||
|
from superset.extensions import security_manager
|
||||||
|
from superset.models.core import Database
|
||||||
|
from superset.models.dashboard import Dashboard
|
||||||
|
from superset.models.helpers import AuditMixinNullable
|
||||||
|
from superset.models.slice import Slice
|
||||||
|
|
||||||
|
metadata = Model.metadata # pylint: disable=no-member
|
||||||
|
|
||||||
|
|
||||||
|
class ReportScheduleType(str, enum.Enum):
|
||||||
|
ALERT = "Alert"
|
||||||
|
REPORT = "Report"
|
||||||
|
|
||||||
|
|
||||||
|
class ReportScheduleValidatorType(str, enum.Enum):
|
||||||
|
""" Validator types for alerts """
|
||||||
|
|
||||||
|
not_null = "not null"
|
||||||
|
operator = "operator"
|
||||||
|
|
||||||
|
|
||||||
|
class ReportRecipientType(str, enum.Enum):
|
||||||
|
EMAIL = "Email"
|
||||||
|
SLACK = "Slack"
|
||||||
|
|
||||||
|
|
||||||
|
class ReportLogState(str, enum.Enum):
|
||||||
|
SUCCESS = "Success"
|
||||||
|
ERROR = "Error"
|
||||||
|
|
||||||
|
|
||||||
|
class ReportEmailFormat(str, enum.Enum):
|
||||||
|
VISUALIZATION = "Visualization"
|
||||||
|
DATA = "Raw data"
|
||||||
|
|
||||||
|
|
||||||
|
report_schedule_user = Table(
|
||||||
|
"report_schedule_user",
|
||||||
|
metadata,
|
||||||
|
Column("id", Integer, primary_key=True),
|
||||||
|
Column("user_id", Integer, ForeignKey("ab_user.id"), nullable=False),
|
||||||
|
Column(
|
||||||
|
"report_schedule_id", Integer, ForeignKey("report_schedule.id"), nullable=False
|
||||||
|
),
|
||||||
|
UniqueConstraint("user_id", "report_schedule_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReportSchedule(Model, AuditMixinNullable):
|
||||||
|
|
||||||
|
"""
|
||||||
|
Report Schedules, supports alerts and reports
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "report_schedule"
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
type = Column(String(50), nullable=False)
|
||||||
|
name = Column(String(150), nullable=False, unique=True)
|
||||||
|
description = Column(Text)
|
||||||
|
context_markdown = Column(Text)
|
||||||
|
active = Column(Boolean, default=True, index=True)
|
||||||
|
crontab = Column(String(50), nullable=False)
|
||||||
|
sql = Column(Text())
|
||||||
|
# (Alerts/Reports) M-O to chart
|
||||||
|
chart_id = Column(Integer, ForeignKey("slices.id"), nullable=True)
|
||||||
|
chart = relationship(Slice, backref="report_schedules", foreign_keys=[chart_id])
|
||||||
|
# (Alerts/Reports) M-O to dashboard
|
||||||
|
dashboard_id = Column(Integer, ForeignKey("dashboards.id"), nullable=True)
|
||||||
|
dashboard = relationship(
|
||||||
|
Dashboard, backref="report_schedules", foreign_keys=[dashboard_id]
|
||||||
|
)
|
||||||
|
# (Alerts) M-O to database
|
||||||
|
database_id = Column(Integer, ForeignKey("dbs.id"), nullable=True)
|
||||||
|
database = relationship(Database, foreign_keys=[database_id])
|
||||||
|
owners = relationship(security_manager.user_model, secondary=report_schedule_user)
|
||||||
|
|
||||||
|
# (Alerts) Stamped last observations
|
||||||
|
last_eval_dttm = Column(DateTime)
|
||||||
|
last_state = Column(String(50))
|
||||||
|
last_value = Column(Float)
|
||||||
|
last_value_row_json = Column(Text)
|
||||||
|
|
||||||
|
# (Alerts) Observed value validation related columns
|
||||||
|
validator_type = Column(String(100))
|
||||||
|
validator_config_json = Column(Text, default="{}")
|
||||||
|
|
||||||
|
# Log retention
|
||||||
|
log_retention = Column(Integer, default=90)
|
||||||
|
grace_period = Column(Integer, default=60 * 60 * 4)
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return str(self.name)
|
||||||
|
|
||||||
|
|
||||||
|
class ReportRecipients(
|
||||||
|
Model, AuditMixinNullable
|
||||||
|
): # pylint: disable=too-few-public-methods
|
||||||
|
|
||||||
|
"""
|
||||||
|
Report Recipients, meant to support multiple notification types, eg: Slack, email
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "report_recipient"
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
type = Column(String(50), nullable=False)
|
||||||
|
recipient_config_json = Column(Text, default="{}")
|
||||||
|
report_schedule_id = Column(
|
||||||
|
Integer, ForeignKey("report_schedule.id"), nullable=False
|
||||||
|
)
|
||||||
|
report_schedule = relationship(
|
||||||
|
ReportSchedule, backref="recipients", foreign_keys=[report_schedule_id]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReportExecutionLog(Model): # pylint: disable=too-few-public-methods
|
||||||
|
|
||||||
|
"""
|
||||||
|
Report Execution Log, hold the result of the report execution with timestamps,
|
||||||
|
last observation and possible error messages
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "report_execution_log"
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
|
||||||
|
# Timestamps
|
||||||
|
scheduled_dttm = Column(DateTime, nullable=False)
|
||||||
|
start_dttm = Column(DateTime)
|
||||||
|
end_dttm = Column(DateTime)
|
||||||
|
|
||||||
|
# (Alerts) Observed values
|
||||||
|
value = Column(Float)
|
||||||
|
value_row_json = Column(Text)
|
||||||
|
|
||||||
|
state = Column(String(50), nullable=False)
|
||||||
|
error_message = Column(Text)
|
||||||
|
|
||||||
|
report_schedule_id = Column(
|
||||||
|
Integer, ForeignKey("report_schedule.id"), nullable=False
|
||||||
|
)
|
||||||
|
report_schedule = relationship(
|
||||||
|
ReportSchedule, backref="logs", foreign_keys=[report_schedule_id]
|
||||||
|
)
|
Loading…
Reference in New Issue
Block a user