feat(echarts): Implement stream graph for Echarts Timeseries (#23410)

superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/controlPanel.tsx

@@ -35,8 +35,9 @@ import {
   showValueControl,
   richTooltipSection,
   seriesOrderSection,
+  percentageThresholdControl,
 } from '../../controls';
-import { AreaChartExtraControlsOptions } from '../../constants';
+import { AreaChartStackControlOptions } from '../../constants';
 
 const {
   logAxis,
@@ -109,13 +110,14 @@ const config: ControlPanelConfig = {
               type: 'SelectControl',
               label: t('Stacked Style'),
               renderTrigger: true,
-              choices: AreaChartExtraControlsOptions,
+              choices: AreaChartStackControlOptions,
               default: null,
               description: t('Stack series on top of each other'),
             },
           },
         ],
         [onlyTotalControl],
+        [percentageThresholdControl],
         [
           {
             name: 'show_extra_controls',

superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/controlPanel.tsx

@@ -29,12 +29,6 @@ import {
   sections,
   sharedControls,
 } from '@superset-ui/chart-controls';
-
-import { OrientationType } from '../../types';
-import {
-  DEFAULT_FORM_DATA,
-  TIME_SERIES_DESCRIPTION_TEXT,
-} from '../../constants';
 import {
   legendSection,
   richTooltipSection,
@@ -42,6 +36,12 @@ import {
   showValueSection,
 } from '../../../controls';
 
+import { OrientationType } from '../../types';
+import {
+  DEFAULT_FORM_DATA,
+  TIME_SERIES_DESCRIPTION_TEXT,
+} from '../../constants';
+
 const {
   logAxis,
   minorSplitLine,

superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts

@@ -43,7 +43,6 @@ import { ZRLineType } from 'echarts/types/src/util/types';
 import {
   EchartsTimeseriesChartProps,
   EchartsTimeseriesFormData,
-  EchartsTimeseriesSeriesType,
   TimeseriesChartTransformedProps,
   OrientationType,
 } from './types';
@@ -74,6 +73,7 @@ import {
 import { convertInteger } from '../utils/convertInteger';
 import { defaultGrid, defaultYAxis } from '../defaults';
 import {
+  getBaselineSeriesForStream,
   getPadding,
   getTooltipTimeFormatter,
   getXAxisFormatter,
@@ -84,7 +84,7 @@ import {
   transformTimeseriesAnnotation,
 } from './transformers';
 import {
-  AreaChartExtraControlsValue,
+  StackControlsValue,
   TIMESERIES_CONSTANTS,
   TIMEGRAIN_TO_TIMESTAMP,
 } from '../constants';
@@ -195,7 +195,6 @@ export default function transformProps(
     fillNeighborValue: stack && !forecastEnabled ? 0 : undefined,
     xAxis: xAxisLabel,
     extraMetricLabels,
-    removeNulls: seriesType === EchartsTimeseriesSeriesType.Scatter,
     stack,
     totalStackedValues,
     isHorizontal,
@@ -210,7 +209,7 @@ export default function transformProps(
   const seriesContexts = extractForecastSeriesContexts(
     Object.values(rawSeries).map(series => series.name as string),
   );
-  const isAreaExpand = stack === AreaChartExtraControlsValue.Expand;
+  const isAreaExpand = stack === StackControlsValue.Expand;
   const xAxisDataType = dataTypes?.[xAxisLabel] ?? dataTypes?.[xAxisOrig];
 
   const xAxisType = getAxisType(xAxisDataType);
@@ -243,9 +242,29 @@ export default function transformProps(
       isHorizontal,
       lineStyle,
     });
-    if (transformedSeries) series.push(transformedSeries);
+    if (transformedSeries) {
+      if (stack === StackControlsValue.Stream) {
+        // bug in Echarts - `stackStrategy: 'all'` doesn't work with nulls, so we cast them to 0
+        series.push({
+          ...transformedSeries,
+          data: (transformedSeries.data as any).map(
+            (row: [string | number, number]) => [row[0], row[1] ?? 0],
+          ),
+        });
+      } else {
+        series.push(transformedSeries);
+      }
+    }
   });
 
+  if (stack === StackControlsValue.Stream) {
+    const baselineSeries = getBaselineSeriesForStream(
+      series.map(entry => entry.data) as [string | number, number][][],
+      seriesType,
+    );
+
+    series.unshift(baselineSeries);
+  }
   const selectedValues = (filterState.selectedValues || []).reduce(
     (acc: Record<string, number>, selectedValue: string) => {
       const index = series.findIndex(({ name }) => name === selectedValue);
@@ -428,6 +447,9 @@ export default function transformProps(
 
         Object.keys(forecastValues).forEach(key => {
           const value = forecastValues[key];
+          if (value.observation === 0 && stack) {
+            return;
+          }
           const content = formatForecastTooltipSeries({
             ...value,
             seriesName: key,

superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformers.ts

@@ -19,6 +19,7 @@
 import {
   AnnotationData,
   AnnotationOpacity,
+  AxisType,
   CategoricalColorScale,
   EventAnnotationLayer,
   FilterState,
@@ -33,7 +34,6 @@ import {
   TimeFormatter,
   TimeseriesAnnotationLayer,
   TimeseriesDataRecord,
-  AxisType,
 } from '@superset-ui/core';
 import { SeriesOption } from 'echarts';
 import {
@@ -53,8 +53,12 @@ import {
 import { MarkLine1DDataItemOption } from 'echarts/types/src/component/marker/MarkLineModel';
 
 import { extractForecastSeriesContext } from '../utils/forecast';
-import { ForecastSeriesEnum, LegendOrientation, StackType } from '../types';
-import { EchartsTimeseriesSeriesType } from './types';
+import {
+  EchartsTimeseriesSeriesType,
+  ForecastSeriesEnum,
+  LegendOrientation,
+  StackType,
+} from '../types';
 
 import {
   evalFormula,
@@ -64,11 +68,79 @@ import {
 } from '../utils/annotation';
 import { currentSeries, getChartPadding } from '../utils/series';
 import {
-  AreaChartExtraControlsValue,
   OpacityEnum,
+  StackControlsValue,
   TIMESERIES_CONSTANTS,
 } from '../constants';
 
+// based on weighted wiggle algorithm
+// source: https://ieeexplore.ieee.org/document/4658136
+export const getBaselineSeriesForStream = (
+  series: [string | number, number][][],
+  seriesType: EchartsTimeseriesSeriesType,
+) => {
+  const seriesLength = series[0].length;
+  const baselineSeriesDelta = new Array(seriesLength).fill([0, 0]);
+  const getVal = (value: number | null) => value ?? 0;
+  for (let i = 0; i < seriesLength; i += 1) {
+    let seriesSum = 0;
+    let weightedSeriesSum = 0;
+    for (let j = 0; j < series.length; j += 1) {
+      const delta =
+        i > 0
+          ? getVal(series[j][i][1]) - getVal(series[j][i - 1][1])
+          : getVal(series[j][i][1]);
+      let deltaPrev = 0;
+      for (let k = 1; k < j - 1; k += 1) {
+        deltaPrev +=
+          i > 0
+            ? getVal(series[k][i][1]) - getVal(series[k][i - 1][1])
+            : getVal(series[k][i][1]);
+      }
+      weightedSeriesSum += (0.5 * delta + deltaPrev) * getVal(series[j][i][1]);
+      seriesSum += getVal(series[j][i][1]);
+    }
+    baselineSeriesDelta[i] = [series[0][i][0], -weightedSeriesSum / seriesSum];
+  }
+  const baselineSeries = baselineSeriesDelta.reduce((acc, curr, i) => {
+    if (i === 0) {
+      acc.push(curr);
+    } else {
+      acc.push([curr[0], acc[i - 1][1] + curr[1]]);
+    }
+    return acc;
+  }, []);
+  return {
+    data: baselineSeries,
+    name: 'baseline',
+    stack: 'obs',
+    stackStrategy: 'all' as const,
+    type: 'line' as const,
+    lineStyle: {
+      opacity: 0,
+    },
+    tooltip: {
+      show: false,
+    },
+    silent: true,
+    showSymbol: false,
+    areaStyle: {
+      opacity: 0,
+    },
+    step: [
+      EchartsTimeseriesSeriesType.Start,
+      EchartsTimeseriesSeriesType.Middle,
+      EchartsTimeseriesSeriesType.End,
+    ].includes(seriesType)
+      ? (seriesType as
+          | EchartsTimeseriesSeriesType.Start
+          | EchartsTimeseriesSeriesType.Middle
+          | EchartsTimeseriesSeriesType.End)
+      : undefined,
+    smooth: seriesType === EchartsTimeseriesSeriesType.Smooth,
+  };
+};
+
 export function transformSeries(
   series: SeriesOption,
   colorScale: CategoricalColorScale,
@@ -190,9 +262,10 @@ export function transformSeries(
       showSymbol = true;
     }
   }
-  const lineStyle = isConfidenceBand
-    ? { ...opts.lineStyle, opacity: OpacityEnum.Transparent }
-    : { ...opts.lineStyle, opacity };
+  const lineStyle =
+    isConfidenceBand || (stack === StackControlsValue.Stream && area)
+      ? { ...opts.lineStyle, opacity: OpacityEnum.Transparent }
+      : { ...opts.lineStyle, opacity };
   return {
     ...series,
     queryIndex,
@@ -208,7 +281,10 @@ export function transformSeries(
       ? seriesType
       : undefined,
     stack: stackId,
-    stackStrategy: isConfidenceBand ? 'all' : 'samesign',
+    stackStrategy:
+      isConfidenceBand || stack === StackControlsValue.Stream
+        ? 'all'
+        : 'samesign',
     lineStyle,
     areaStyle:
       area || forecastSeries.type === ForecastSeriesEnum.ForecastUpper
@@ -234,7 +310,7 @@ export function transformSeries(
         const { value, dataIndex, seriesIndex, seriesName } = params;
         const numericValue = isHorizontal ? value[0] : value[1];
         const isSelectedLegend = currentSeries.legend === seriesName;
-        const isAreaExpand = stack === AreaChartExtraControlsValue.Expand;
+        const isAreaExpand = stack === StackControlsValue.Expand;
         if (!formatter) return numericValue;
         if (!stack || isSelectedLegend) return formatter(numericValue);
         if (!onlyTotal) {

superset-frontend/plugins/plugin-chart-echarts/src/components/ExtraControls.tsx

@@ -22,7 +22,7 @@ import {
   RadioButtonOption,
   sharedControlComponents,
 } from '@superset-ui/chart-controls';
-import { AreaChartExtraControlsOptions } from '../constants';
+import { AreaChartStackControlOptions } from '../constants';
 
 const { RadioButtonControl } = sharedControlComponents;
 
@@ -53,7 +53,7 @@ export function useExtraControl<
 
   const extraControlsOptions = useMemo(() => {
     if (area) {
-      return AreaChartExtraControlsOptions;
+      return AreaChartStackControlOptions;
     }
     return [];
   }, [area]);

superset-frontend/plugins/plugin-chart-echarts/src/constants.ts

@@ -71,20 +71,26 @@ export enum OpacityEnum {
   NonTransparent = 1,
 }
 
-export enum AreaChartExtraControlsValue {
+export enum StackControlsValue {
   Stack = 'Stack',
+  Stream = 'Stream',
   Expand = 'Expand',
 }
 
-export const AreaChartExtraControlsOptions: [
+export const StackControlOptions: [
   JsonValue,
   Exclude<ReactNode, null | undefined | boolean>,
 ][] = [
   [null, t('None')],
-  [AreaChartExtraControlsValue.Stack, t('Stack')],
-  [AreaChartExtraControlsValue.Expand, t('Expand')],
+  [StackControlsValue.Stack, t('Stack')],
+  [StackControlsValue.Stream, t('Stream')],
 ];
 
+export const AreaChartStackControlOptions: [
+  JsonValue,
+  Exclude<ReactNode, null | undefined | boolean>,
+][] = [...StackControlOptions, [StackControlsValue.Expand, t('Expand')]];
+
 export const TIMEGRAIN_TO_TIMESTAMP = {
   [TimeGranularity.HOUR]: 3600 * 1000,
   [TimeGranularity.DAY]: 3600 * 1000 * 24,

superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx

@@ -27,6 +27,7 @@ import {
 import {
   DEFAULT_LEGEND_FORM_DATA,
   DEFAULT_SORT_SERIES_DATA,
+  StackControlOptions,
 } from './constants';
 import { DEFAULT_FORM_DATA } from './Timeseries/constants';
 import { SortSeriesType } from './types';
@@ -119,10 +120,11 @@ export const showValueControl: ControlSetItem = {
 export const stackControl: ControlSetItem = {
   name: 'stack',
   config: {
-    type: 'CheckboxControl',
-    label: t('Stack series'),
+    type: 'SelectControl',
+    label: t('Stacked Style'),
     renderTrigger: true,
-    default: false,
+    choices: StackControlOptions,
+    default: null,
     description: t('Stack series on top of each other'),
   },
 };
@@ -142,7 +144,7 @@ export const onlyTotalControl: ControlSetItem = {
   },
 };
 
-const percentageThresholdControl: ControlSetItem = {
+export const percentageThresholdControl: ControlSetItem = {
   name: 'percentage_threshold',
   config: {
     type: 'TextControl',

superset-frontend/plugins/plugin-chart-echarts/src/types.ts

@@ -29,7 +29,7 @@ import {
 } from '@superset-ui/core';
 import { EChartsCoreOption, ECharts } from 'echarts';
 import { TooltipMarker } from 'echarts/types/src/util/format';
-import { AreaChartExtraControlsValue } from './constants';
+import { StackControlsValue } from './constants';
 
 export type EchartsStylesProps = {
   height: number;
@@ -159,7 +159,7 @@ export interface TitleFormData {
   yAxisTitlePosition: string;
 }
 
-export type StackType = boolean | null | Partial<AreaChartExtraControlsValue>;
+export type StackType = boolean | null | Partial<StackControlsValue>;
 
 export interface TreePathInfo {
   name: string;

superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts

@@ -32,7 +32,7 @@ import {
 import { format, LegendComponentOption, SeriesOption } from 'echarts';
 import { sumBy, meanBy, minBy, maxBy, orderBy } from 'lodash';
 import {
-  AreaChartExtraControlsValue,
+  StackControlsValue,
   NULL_STRING,
   TIMESERIES_CONSTANTS,
 } from '../constants';
@@ -207,7 +207,7 @@ export function extractSeries(
         if (isFillNeighborValue) {
           value = fillNeighborValue;
         } else if (
-          stack === AreaChartExtraControlsValue.Expand &&
+          stack === StackControlsValue.Expand &&
           totalStackedValues.length > 0
         ) {
           value = ((value || 0) as number) / totalStackedValues[idx];

superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts

@@ -295,6 +295,112 @@ describe('EchartsTimeseries transformProps', () => {
       }),
     );
   });
+
+  it('Should add a baseline series for stream graph', () => {
+    const streamQueriesData = [
+      {
+        data: [
+          {
+            'San Francisco': 120,
+            'New York': 220,
+            Boston: 150,
+            Miami: 270,
+            Denver: 800,
+            __timestamp: 599616000000,
+          },
+          {
+            'San Francisco': 150,
+            'New York': 190,
+            Boston: 240,
+            Miami: 350,
+            Denver: 700,
+            __timestamp: 599616000001,
+          },
+          {
+            'San Francisco': 130,
+            'New York': 300,
+            Boston: 250,
+            Miami: 410,
+            Denver: 650,
+            __timestamp: 599616000002,
+          },
+          {
+            'San Francisco': 90,
+            'New York': 340,
+            Boston: 300,
+            Miami: 480,
+            Denver: 590,
+            __timestamp: 599616000003,
+          },
+          {
+            'San Francisco': 260,
+            'New York': 200,
+            Boston: 420,
+            Miami: 490,
+            Denver: 760,
+            __timestamp: 599616000004,
+          },
+          {
+            'San Francisco': 250,
+            'New York': 250,
+            Boston: 380,
+            Miami: 360,
+            Denver: 400,
+            __timestamp: 599616000005,
+          },
+          {
+            'San Francisco': 160,
+            'New York': 210,
+            Boston: 330,
+            Miami: 440,
+            Denver: 580,
+            __timestamp: 599616000006,
+          },
+        ],
+      },
+    ];
+    const streamFormData = { ...formData, stack: 'Stream' };
+    const props = {
+      ...chartPropsConfig,
+      formData: streamFormData,
+      queriesData: streamQueriesData,
+    };
+
+    const chartProps = new ChartProps(props);
+    expect(
+      (
+        transformProps(chartProps as EchartsTimeseriesChartProps).echartOptions
+          .series as any[]
+      )[0],
+    ).toEqual({
+      areaStyle: {
+        opacity: 0,
+      },
+      lineStyle: {
+        opacity: 0,
+      },
+      name: 'baseline',
+      showSymbol: false,
+      silent: true,
+      smooth: false,
+      stack: 'obs',
+      stackStrategy: 'all',
+      step: undefined,
+      tooltip: {
+        show: false,
+      },
+      type: 'line',
+      data: [
+        [599616000000, -415.7692307692308],
+        [599616000001, -403.6219915054271],
+        [599616000002, -476.32314093071443],
+        [599616000003, -514.2120298196033],
+        [599616000004, -485.7378514158475],
+        [599616000005, -419.6402904402378],
+        [599616000006, -442.9833136960517],
+      ],
+    });
+  });
 });
 
 describe('Does transformProps transform series correctly', () => {

superset/migrations/versions/2023-03-17_13-24_b5ea9d343307_bar_chart_stack_options.py

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""bar_chart_stack_options
+
+Revision ID: b5ea9d343307
+Revises: d0ac08bb5b83
+Create Date: 2023-03-17 13:24:54.662754
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "b5ea9d343307"
+down_revision = "d0ac08bb5b83"
+
+import json
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import and_, Column, Integer, String, Text
+from sqlalchemy.ext.declarative import declarative_base
+
+from superset import db
+
+Base = declarative_base()
+
+CHART_TYPE = "%echarts_timeseries%"
+
+
+class Slice(Base):
+    """Declarative class to do query in upgrade"""
+
+    __tablename__ = "slices"
+    id = Column(Integer, primary_key=True)
+    viz_type = Column(String(250))
+    params = Column(Text)
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    slices = session.query(Slice).filter(Slice.viz_type.like(CHART_TYPE)).all()
+    for slc in slices:
+        try:
+            params = json.loads(slc.params)
+            stack = params.get("stack", None)
+            if stack:
+                params["stack"] = "Stack"
+            else:
+                params["stack"] = None
+            slc.params = json.dumps(params, sort_keys=True)
+        except Exception as e:
+            print(e)
+            print(f"Parsing params for slice {slc.id} failed.")
+            pass
+
+    session.commit()
+    session.close()
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    slices = session.query(Slice).filter(Slice.viz_type.like(CHART_TYPE)).all()
+    for slc in slices:
+        try:
+            params = json.loads(slc.params)
+            stack = params.get("stack", None)
+            if stack == "Stack" or stack == "Stream":
+                params["stack"] = True
+            else:
+                params["stack"] = False
+            slc.params = json.dumps(params, sort_keys=True)
+        except Exception as e:
+            print(e)
+            print(f"Parsing params for slice {slc.id} failed.")
+            pass
+
+    session.commit()
+    session.close()