feat: Adds Area chart migration logic (#25952)

This commit is contained in:
Michael S. Molina 2023-11-16 14:28:09 -03:00 committed by GitHub
parent 7f0c3b20ad
commit 97d89d7340
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 330 additions and 276 deletions

View File

@ -17,4 +17,4 @@
[pytest]
testpaths =
tests
python_files = *_test.py test_*.py *_tests.py
python_files = *_test.py test_*.py *_tests.py *viz/utils.py

View File

@ -75,7 +75,6 @@ def migrate_chart(config: dict[str, Any]) -> dict[str, Any]:
if isclass(class_)
and issubclass(class_, MigrateViz)
and hasattr(class_, "source_viz_type")
and class_ != processors.MigrateAreaChart # incomplete
}
output = copy.deepcopy(config)

View File

@ -160,9 +160,7 @@ class MigrateViz:
slices = session.query(Slice).filter(Slice.viz_type == cls.source_viz_type)
for slc in paginated_update(
slices,
lambda current, total: print(
f" Updating {current}/{total} charts", end="\r"
),
lambda current, total: print(f"Upgraded {current}/{total} charts"),
):
new_viz = cls.upgrade_slice(slc)
session.merge(new_viz)
@ -177,9 +175,7 @@ class MigrateViz:
)
for slc in paginated_update(
slices,
lambda current, total: print(
f" Downgrading {current}/{total} charts", end="\r"
),
lambda current, total: print(f"Downgraded {current}/{total} charts"),
):
new_viz = cls.downgrade_slice(slc)
session.merge(new_viz)

View File

@ -36,40 +36,6 @@ class MigrateTreeMap(MigrateViz):
self.data["metric"] = self.data["metrics"][0]
class MigrateAreaChart(MigrateViz):
"""
Migrate area charts.
This migration is incomplete, see https://github.com/apache/superset/pull/24703#discussion_r1265222611
for more details. If you fix this migration, please update the ``migrate_chart``
function in ``superset/charts/commands/importers/v1/utils.py`` so that it gets
applied in chart imports.
"""
source_viz_type = "area"
target_viz_type = "echarts_area"
remove_keys = {"contribution", "stacked_style", "x_axis_label"}
def _pre_action(self) -> None:
if self.data.get("contribution"):
self.data["contributionMode"] = "row"
if stacked := self.data.get("stacked_style"):
stacked_map = {
"expand": "Expand",
"stack": "Stack",
}
self.data["show_extra_controls"] = True
self.data["stack"] = stacked_map.get(stacked)
if x_axis := self.data.get("granularity_sqla"):
self.data["x_axis"] = x_axis
if x_axis_label := self.data.get("x_axis_label"):
self.data["x_axis_title"] = x_axis_label
self.data["x_axis_title_margin"] = 30
class MigratePivotTable(MigrateViz):
source_viz_type = "pivot_table"
target_viz_type = "pivot_table_v2"
@ -137,10 +103,22 @@ class MigrateSunburst(MigrateViz):
class TimeseriesChart(MigrateViz):
has_x_axis_control = True
rename_keys = {
"bottom_margin": "x_axis_title_margin",
"left_margin": "y_axis_title_margin",
"show_controls": "show_extra_controls",
"x_axis_label": "x_axis_title",
"x_axis_format": "x_axis_time_format",
"x_ticks_layout": "xAxisLabelRotation",
"y_axis_label": "y_axis_title",
"y_axis_showminmax": "truncateYAxis",
"y_log_scale": "logAxis",
}
remove_keys = {"contribution", "show_brush", "show_markers"}
def _pre_action(self) -> None:
self.data["contributionMode"] = "row" if self.data.get("contribution") else None
self.data["zoomable"] = self.data.get("show_brush") != "no"
self.data["zoomable"] = self.data.get("show_brush") == "yes"
self.data["markerEnabled"] = self.data.get("show_markers") or False
self.data["y_axis_showminmax"] = True
@ -163,23 +141,19 @@ class TimeseriesChart(MigrateViz):
"difference" if comparison_type == "absolute" else comparison_type
)
if x_ticks_layout := self.data.get("x_ticks_layout"):
self.data["x_ticks_layout"] = 45 if x_ticks_layout == "45°" else 0
class MigrateLineChart(TimeseriesChart):
source_viz_type = "line"
target_viz_type = "echarts_timeseries_line"
rename_keys = {
"x_axis_label": "x_axis_title",
"bottom_margin": "x_axis_title_margin",
"x_axis_format": "x_axis_time_format",
"y_axis_label": "y_axis_title",
"left_margin": "y_axis_title_margin",
"y_axis_showminmax": "truncateYAxis",
"y_log_scale": "logAxis",
}
def _pre_action(self) -> None:
super()._pre_action()
self.remove_keys.add("line_interpolation")
line_interpolation = self.data.get("line_interpolation")
if line_interpolation == "cardinal":
self.target_viz_type = "echarts_timeseries_smooth"
@ -189,3 +163,24 @@ class MigrateLineChart(TimeseriesChart):
elif line_interpolation == "step-after":
self.target_viz_type = "echarts_timeseries_step"
self.data["seriesType"] = "end"
class MigrateAreaChart(TimeseriesChart):
source_viz_type = "area"
target_viz_type = "echarts_area"
stacked_map = {
"expand": "Expand",
"stack": "Stack",
"stream": "Stream",
}
def _pre_action(self) -> None:
super()._pre_action()
self.remove_keys.add("stacked_style")
self.data["stack"] = self.stacked_map.get(
self.data.get("stacked_style") or "stack"
)
self.data["opacity"] = 0.7

View File

@ -1,99 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
from superset.app import SupersetApp
from superset.migrations.shared.migrate_viz import MigrateAreaChart
area_form_data = """{
"adhoc_filters": [],
"annotation_layers": [],
"bottom_margin": "auto",
"color_scheme": "lyftColors",
"comparison_type": "values",
"contribution": true,
"datasource": "2__table",
"extra_form_data": {},
"granularity_sqla": "ds",
"groupby": [
"gender"
],
"line_interpolation": "linear",
"metrics": [
"sum__num"
],
"order_desc": true,
"rich_tooltip": true,
"rolling_type": "None",
"row_limit": 10000,
"show_brush": "auto",
"show_controls": true,
"show_legend": true,
"slice_id": 165,
"stacked_style": "stack",
"time_grain_sqla": "P1D",
"time_range": "No filter",
"viz_type": "area",
"x_axis_format": "smart_date",
"x_axis_label": "x asix label",
"x_axis_showminmax": false,
"x_ticks_layout": "auto",
"y_axis_bounds": [
null,
null
],
"y_axis_format": "SMART_NUMBER"
}
"""
def test_area_migrate(app_context: SupersetApp) -> None:
from superset.models.slice import Slice
slc = Slice(
viz_type=MigrateAreaChart.source_viz_type,
datasource_type="table",
params=area_form_data,
query_context=f'{{"form_data": {area_form_data}}}',
)
slc = MigrateAreaChart.upgrade_slice(slc)
assert slc.viz_type == MigrateAreaChart.target_viz_type
# verify form_data
new_form_data = json.loads(slc.params)
assert new_form_data["contributionMode"] == "row"
assert "contribution" not in new_form_data
assert new_form_data["show_extra_controls"] is True
assert new_form_data["stack"] == "Stack"
assert new_form_data["x_axis_title"] == "x asix label"
assert new_form_data["x_axis_title_margin"] == 30
assert json.dumps(new_form_data["form_data_bak"], sort_keys=True) == json.dumps(
json.loads(area_form_data), sort_keys=True
)
# verify query_context
new_query_context = json.loads(slc.query_context)
assert (
new_query_context["form_data"]["viz_type"] == MigrateAreaChart.target_viz_type
)
# downgrade
slc = MigrateAreaChart.downgrade_slice(slc)
assert slc.viz_type == MigrateAreaChart.source_viz_type
assert json.dumps(json.loads(slc.params), sort_keys=True) == json.dumps(
json.loads(area_form_data), sort_keys=True
)

View File

@ -31,13 +31,21 @@ def test_migrate_chart_area() -> None:
"description": None,
"certified_by": None,
"certification_details": None,
"viz_type": "area",
"viz_type": "echarts_area",
"query_context": None,
"params": json.dumps(
{
"adhoc_filters": [],
"adhoc_filters": [
{
"clause": "WHERE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "No filter",
"expressionType": "SIMPLE",
}
],
"annotation_layers": [],
"bottom_margin": "auto",
"x_axis_title_margin": "auto",
"color_scheme": "supersetColors",
"comparison_type": "values",
"dashboards": [],

View File

@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
from typing import Any
from superset.migrations.shared.migrate_viz import MigrateDualLine
from tests.unit_tests.migrations.viz.utils import migrate_and_assert
ADHOC_FILTERS = [
{
@ -28,7 +29,7 @@ ADHOC_FILTERS = [
}
]
SOURCE_FORM_DATA = {
SOURCE_FORM_DATA: dict[str, Any] = {
"metric": "num_boys",
"y_axis_format": ",d",
"y_axis_bounds": [50, 100],
@ -42,7 +43,7 @@ SOURCE_FORM_DATA = {
"yAxisIndex": 0,
}
TARGET_FORM_DATA = {
TARGET_FORM_DATA: dict[str, Any] = {
"metrics": ["num_boys"],
"y_axis_format": ",d",
"y_axis_bounds": [50, 100],
@ -64,34 +65,4 @@ TARGET_FORM_DATA = {
def test_migration() -> None:
source = SOURCE_FORM_DATA.copy()
target = TARGET_FORM_DATA.copy()
upgrade_downgrade(source, target)
def upgrade_downgrade(source, target) -> None:
from superset.models.slice import Slice
dumped_form_data = json.dumps(source)
slc = Slice(
viz_type=MigrateDualLine.source_viz_type,
datasource_type="table",
params=dumped_form_data,
query_context=f'{{"form_data": {dumped_form_data}}}',
)
# upgrade
slc = MigrateDualLine.upgrade_slice(slc)
# verify form_data
new_form_data = json.loads(slc.params)
assert new_form_data == target
assert new_form_data["form_data_bak"] == source
# verify query_context
new_query_context = json.loads(slc.query_context)
assert new_query_context["form_data"]["viz_type"] == "mixed_timeseries"
# downgrade
slc = MigrateDualLine.downgrade_slice(slc)
assert slc.viz_type == MigrateDualLine.source_viz_type
assert json.loads(slc.params) == source
migrate_and_assert(MigrateDualLine, source, target)

View File

@ -0,0 +1,42 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Any
from superset.migrations.shared.migrate_viz import MigrateAreaChart
from tests.unit_tests.migrations.viz.utils import (
migrate_and_assert,
TIMESERIES_SOURCE_FORM_DATA,
TIMESERIES_TARGET_FORM_DATA,
)
SOURCE_FORM_DATA: dict[str, Any] = {
"viz_type": "area",
"stacked_style": "stream",
}
TARGET_FORM_DATA: dict[str, Any] = {
"form_data_bak": SOURCE_FORM_DATA,
"viz_type": "echarts_area",
"opacity": 0.7,
"stack": "Stream",
}
def test_migration() -> None:
SOURCE_FORM_DATA.update(TIMESERIES_SOURCE_FORM_DATA)
TARGET_FORM_DATA.update(TIMESERIES_TARGET_FORM_DATA)
migrate_and_assert(MigrateAreaChart, SOURCE_FORM_DATA, TARGET_FORM_DATA)

View File

@ -0,0 +1,39 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Any
from superset.migrations.shared.migrate_viz import MigrateLineChart
from tests.unit_tests.migrations.viz.utils import (
migrate_and_assert,
TIMESERIES_SOURCE_FORM_DATA,
TIMESERIES_TARGET_FORM_DATA,
)
SOURCE_FORM_DATA: dict[str, Any] = {
"viz_type": "line",
}
TARGET_FORM_DATA: dict[str, Any] = {
"form_data_bak": SOURCE_FORM_DATA,
"viz_type": "echarts_timeseries_line",
}
def test_migration() -> None:
SOURCE_FORM_DATA.update(TIMESERIES_SOURCE_FORM_DATA)
TARGET_FORM_DATA.update(TIMESERIES_TARGET_FORM_DATA)
migrate_and_assert(MigrateLineChart, SOURCE_FORM_DATA, TARGET_FORM_DATA)

View File

@ -14,122 +14,40 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
from typing import Any
from superset.migrations.shared.migrate_viz import MigratePivotTable
from tests.unit_tests.conftest import with_feature_flags
from tests.unit_tests.migrations.viz.utils import migrate_and_assert
SOURCE_FORM_DATA = {
"adhoc_filters": [],
SOURCE_FORM_DATA: dict[str, Any] = {
"any_other_key": "untouched",
"columns": ["state"],
"combine_metric": True,
"granularity_sqla": "ds",
"groupby": ["name"],
"number_format": "SMART_NUMBER",
"pandas_aggfunc": "sum",
"pivot_margins": True,
"time_range": "100 years ago : now",
"timeseries_limit_metric": "count",
"transpose_pivot": True,
"viz_type": "pivot_table",
}
TARGET_FORM_DATA = {
"adhoc_filters": [],
TARGET_FORM_DATA: dict[str, Any] = {
"any_other_key": "untouched",
"aggregateFunction": "Sum",
"colTotals": True,
"colSubTotals": True,
"combineMetric": True,
"form_data_bak": SOURCE_FORM_DATA,
"granularity_sqla": "ds",
"groupbyColumns": ["state"],
"groupbyRows": ["name"],
"rowOrder": "value_z_to_a",
"series_limit_metric": "count",
"time_range": "100 years ago : now",
"transposePivot": True,
"valueFormat": "SMART_NUMBER",
"viz_type": "pivot_table_v2",
}
@with_feature_flags(GENERIC_CHART_AXES=False)
def test_migration_without_generic_chart_axes() -> None:
source = SOURCE_FORM_DATA.copy()
target = TARGET_FORM_DATA.copy()
upgrade_downgrade(source, target)
@with_feature_flags(GENERIC_CHART_AXES=True)
def test_migration_with_generic_chart_axes() -> None:
source = SOURCE_FORM_DATA.copy()
target = TARGET_FORM_DATA.copy()
target["adhoc_filters"] = [
{
"clause": "WHERE",
"comparator": "100 years ago : now",
"expressionType": "SIMPLE",
"operator": "TEMPORAL_RANGE",
"subject": "ds",
}
]
target.pop("granularity_sqla")
target.pop("time_range")
upgrade_downgrade(source, target)
@with_feature_flags(GENERIC_CHART_AXES=True)
def test_custom_sql_time_column() -> None:
source = SOURCE_FORM_DATA.copy()
source["granularity_sqla"] = {
"expressionType": "SQL",
"label": "ds",
"sqlExpression": "sum(ds)",
}
target = TARGET_FORM_DATA.copy()
target["adhoc_filters"] = [
{
"clause": "WHERE",
"comparator": None,
"expressionType": "SQL",
"operator": "TEMPORAL_RANGE",
"sqlExpression": "sum(ds)",
"subject": "ds",
}
]
target["form_data_bak"] = source
target.pop("granularity_sqla")
target.pop("time_range")
upgrade_downgrade(source, target)
def upgrade_downgrade(source, target) -> None:
from superset.models.slice import Slice
dumped_form_data = json.dumps(source)
slc = Slice(
viz_type=MigratePivotTable.source_viz_type,
datasource_type="table",
params=dumped_form_data,
query_context=f'{{"form_data": {dumped_form_data}}}',
)
# upgrade
slc = MigratePivotTable.upgrade_slice(slc)
# verify form_data
new_form_data = json.loads(slc.params)
assert new_form_data == target
assert new_form_data["form_data_bak"] == source
# verify query_context
new_query_context = json.loads(slc.query_context)
assert new_query_context["form_data"]["viz_type"] == "pivot_table_v2"
# downgrade
slc = MigratePivotTable.downgrade_slice(slc)
assert slc.viz_type == MigratePivotTable.source_viz_type
assert json.loads(slc.params) == source
def test_migration() -> None:
migrate_and_assert(MigratePivotTable, SOURCE_FORM_DATA, TARGET_FORM_DATA)

View File

@ -0,0 +1,89 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Any
from superset.migrations.shared.migrate_viz import MigratePivotTable
from tests.unit_tests.conftest import with_feature_flags
from tests.unit_tests.migrations.viz.utils import migrate_and_assert
SOURCE_FORM_DATA: dict[str, Any] = {
"granularity_sqla": "ds",
"time_range": "100 years ago : now",
"viz_type": "pivot_table",
}
TARGET_FORM_DATA: dict[str, Any] = {
"form_data_bak": SOURCE_FORM_DATA,
"granularity_sqla": "ds",
"rowOrder": "value_z_to_a",
"time_range": "100 years ago : now",
"viz_type": "pivot_table_v2",
}
@with_feature_flags(GENERIC_CHART_AXES=False)
def test_migration_without_generic_chart_axes() -> None:
source = SOURCE_FORM_DATA.copy()
target = TARGET_FORM_DATA.copy()
upgrade_downgrade(source, target)
@with_feature_flags(GENERIC_CHART_AXES=True)
def test_migration_with_generic_chart_axes() -> None:
source = SOURCE_FORM_DATA.copy()
target = TARGET_FORM_DATA.copy()
target["adhoc_filters"] = [
{
"clause": "WHERE",
"comparator": "100 years ago : now",
"expressionType": "SIMPLE",
"operator": "TEMPORAL_RANGE",
"subject": "ds",
}
]
target.pop("granularity_sqla")
target.pop("time_range")
upgrade_downgrade(source, target)
@with_feature_flags(GENERIC_CHART_AXES=True)
def test_custom_sql_time_column() -> None:
source = SOURCE_FORM_DATA.copy()
source["granularity_sqla"] = {
"expressionType": "SQL",
"label": "ds",
"sqlExpression": "sum(ds)",
}
target = TARGET_FORM_DATA.copy()
target["adhoc_filters"] = [
{
"clause": "WHERE",
"comparator": None,
"expressionType": "SQL",
"operator": "TEMPORAL_RANGE",
"sqlExpression": "sum(ds)",
"subject": "ds",
}
]
target["form_data_bak"] = source
target.pop("granularity_sqla")
target.pop("time_range")
upgrade_downgrade(source, target)
def upgrade_downgrade(source, target) -> None:
migrate_and_assert(MigratePivotTable, source, target)

View File

@ -0,0 +1,96 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
from typing import Any
from superset.migrations.shared.migrate_viz import MigrateViz
TIMESERIES_SOURCE_FORM_DATA: dict[str, Any] = {
"bottom_margin": 20,
"comparison_type": "absolute",
"contribution": True,
"left_margin": 20,
"rich_tooltip": True,
"rolling_type": "sum",
"show_brush": "yes",
"show_controls": True,
"show_legend": True,
"show_markers": True,
"time_compare": "1 year",
"x_axis_label": "x",
"x_axis_format": "SMART_DATE",
"x_ticks_layout": "45°",
"y_axis_bounds": [0, 100],
"y_axis_format": "SMART_NUMBER",
"y_axis_label": "y",
"y_axis_showminmax": True,
"y_log_scale": True,
}
TIMESERIES_TARGET_FORM_DATA: dict[str, Any] = {
"comparison_type": "difference",
"contributionMode": "row",
"logAxis": True,
"markerEnabled": True,
"rich_tooltip": True,
"rolling_type": "sum",
"show_extra_controls": True,
"show_legend": True,
"time_compare": ["1 year ago"],
"truncateYAxis": True,
"x_axis_title_margin": 20,
"y_axis_title_margin": 20,
"x_axis_title": "x",
"x_axis_time_format": "SMART_DATE",
"xAxisLabelRotation": 45,
"y_axis_bounds": [0, 100],
"y_axis_format": "SMART_NUMBER",
"y_axis_title": "y",
"zoomable": True,
}
def migrate_and_assert(
cls: type[MigrateViz], source: dict[str, Any], target: dict[str, Any]
) -> None:
from superset.models.slice import Slice
dumped_form_data = json.dumps(source)
slc = Slice(
viz_type=cls.source_viz_type,
datasource_type="table",
params=dumped_form_data,
query_context=f'{{"form_data": {dumped_form_data}}}',
)
# upgrade
slc = cls.upgrade_slice(slc)
# verify form_data
new_form_data = json.loads(slc.params)
assert new_form_data == target
assert new_form_data["form_data_bak"] == source
# verify query_context
new_query_context = json.loads(slc.query_context)
assert new_query_context["form_data"]["viz_type"] == cls.target_viz_type
# downgrade
slc = cls.downgrade_slice(slc)
assert slc.viz_type == cls.source_viz_type
assert json.loads(slc.params) == source