feat(explore): adhoc column expressions [ID-3] (#17379)

* add support for adhoc columns to api and sqla model

* fix some types

* fix duplicates in column names

* fix more lint

* fix schema and dedup

* clean up some logic

* first pass at fixing viz.py

* Add frontend support for adhoc columns

* Add title edit

* Fix showing custom title

* Use column name as default value in sql editor

* fix: Adds a loading message when needed in the Select component (#16531)

* fix(tests): make parquet select deterministic with order by (#16570)

* bump emotion to help with cache clobbering (#16559)

* fix: Support Jinja template functions in global async queries (#16412)

* Support Jinja template functions in async queries

* Pylint

* Add tests for async tasks

* Remove redundant has_request_context check

* fix: impersonate user label/tooltip (#16573)

* docs: update for small typos (#16568)

* feat: Add Aurora Data API engine spec (#16535)

* feat: Add Aurora Data API engine spec

* Fix lint

* refactor: sql_json view endpoint: encapsulate ctas parameters (#16548)

* refactor sql_json view endpoint: encapsulate ctas parameters

* fix failed tests

* fix failed tests and ci issues

* refactor sql_json view endpoint: separate concern into ad hod method (#16595)

* feat: Experimental cross-filter plugins (#16594)

* fix:fix get permission function

* feat: add cross filter chart in charts gallery under FF

* chore(deps): bump superset-ui to 0.18.2 (#16601)

* update type guard references

* fix imports

* update series_columns schema

* Add changes that got lost in rebase

* Use current columns name or expression as sql editor init value

* add integration test and do minor fixes

* Bump superset-ui

* fix linting issue

* bump superset-ui to 0.18.22

* resolve merge conflict

* lint

* fix select filter infinite loop

* bump superset-ui to 0.18.23

* Fix auto setting column popover title

* Enable adhoc columns only if UX_BETA enabled

* put back removed test

* Move popover height and width to constants

* Refactor big ternary expression

Co-authored-by: Kamil Gabryjelski <kamil.gabryjelski@gmail.com>
Co-authored-by: Michael S. Molina <70410625+michael-s-molina@users.noreply.github.com>
Co-authored-by: Elizabeth Thompson <eschutho@gmail.com>
Co-authored-by: Rob DiCiuccio <rob.diciuccio@gmail.com>
Co-authored-by: Beto Dealmeida <roberto@dealmeida.net>
Co-authored-by: joeADSP <75027008+joeADSP@users.noreply.github.com>
Co-authored-by: ofekisr <35701650+ofekisr@users.noreply.github.com>
Co-authored-by: simcha90 <56388545+simcha90@users.noreply.github.com>

superset-frontend/package.json

@@ -68,35 +68,35 @@
     "@emotion/cache": "^11.4.0",
     "@emotion/react": "^11.4.1",
     "@emotion/styled": "^11.3.0",
-    "@superset-ui/chart-controls": "^0.18.20",
+    "@superset-ui/chart-controls": "^0.18.23",
-    "@superset-ui/core": "^0.18.20",
+    "@superset-ui/core": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-calendar": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-calendar": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-chord": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-chord": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-country-map": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-country-map": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-event-flow": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-event-flow": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-force-directed": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-force-directed": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-heatmap": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-heatmap": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-histogram": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-histogram": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-horizon": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-horizon": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-map-box": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-map-box": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-paired-t-test": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-paired-t-test": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-parallel-coordinates": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-parallel-coordinates": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-partition": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-partition": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-pivot-table": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-pivot-table": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-rose": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-rose": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-sankey": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-sankey": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-sankey-loop": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-sankey-loop": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-sunburst": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-sunburst": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-treemap": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-treemap": "^0.18.23",
-    "@superset-ui/legacy-plugin-chart-world-map": "^0.18.20",
+    "@superset-ui/legacy-plugin-chart-world-map": "^0.18.23",
-    "@superset-ui/legacy-preset-chart-big-number": "^0.18.20",
+    "@superset-ui/legacy-preset-chart-big-number": "^0.18.23",
     "@superset-ui/legacy-preset-chart-deckgl": "^0.4.13",
-    "@superset-ui/legacy-preset-chart-nvd3": "^0.18.20",
+    "@superset-ui/legacy-preset-chart-nvd3": "^0.18.23",
-    "@superset-ui/plugin-chart-echarts": "^0.18.20",
+    "@superset-ui/plugin-chart-echarts": "^0.18.23",
-    "@superset-ui/plugin-chart-pivot-table": "^0.18.20",
+    "@superset-ui/plugin-chart-pivot-table": "^0.18.23",
-    "@superset-ui/plugin-chart-table": "^0.18.20",
+    "@superset-ui/plugin-chart-table": "^0.18.23",
-    "@superset-ui/plugin-chart-word-cloud": "^0.18.20",
+    "@superset-ui/plugin-chart-word-cloud": "^0.18.23",
-    "@superset-ui/preset-chart-xy": "^0.18.20",
+    "@superset-ui/preset-chart-xy": "^0.18.23",
     "@vx/responsive": "^0.0.195",
     "abortcontroller-polyfill": "^1.1.9",
     "antd": "^4.9.4",

superset-frontend/src/explore/components/controls/DndColumnSelectControl/ColumnSelectPopover.tsx

@@ -17,15 +17,27 @@
  * under the License.
  */
 /* eslint-disable camelcase */
-import React, { useCallback, useMemo, useState } from 'react';
+import React, {
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+import { AdhocColumn, t, styled, css } from '@superset-ui/core';
+import {
+  ColumnMeta,
+  isAdhocColumn,
+  isSavedExpression,
+} from '@superset-ui/chart-controls';
 import Tabs from 'src/components/Tabs';
 import Button from 'src/components/Button';
 import { Select } from 'src/components';
-import { t, styled } from '@superset-ui/core';
 
 import { Form, FormItem } from 'src/components/Form';
+import { SQLEditor } from 'src/components/AsyncAceEditor';
 import { StyledColumnOption } from 'src/explore/components/optionRenderers';
-import { ColumnMeta } from '@superset-ui/chart-controls';
+import { POPOVER_INITIAL_HEIGHT } from 'src/explore/constants';
 
 const StyledSelect = styled(Select)`
   .metric-option {
@@ -41,29 +53,58 @@ const StyledSelect = styled(Select)`
 
 interface ColumnSelectPopoverProps {
   columns: ColumnMeta[];
-  editedColumn?: ColumnMeta;
+  editedColumn?: ColumnMeta | AdhocColumn;
-  onChange: (column: ColumnMeta) => void;
+  onChange: (column: ColumnMeta | AdhocColumn) => void;
   onClose: () => void;
+  setLabel: (title: string) => void;
+  getCurrentTab: (tab: string) => void;
+  label: string;
+  isAdhocColumnsEnabled: boolean;
 }
 
+const getInitialColumnValues = (
+  editedColumn?: ColumnMeta | AdhocColumn,
+): [AdhocColumn?, ColumnMeta?, ColumnMeta?] => {
+  if (!editedColumn) {
+    return [undefined, undefined, undefined];
+  }
+  if (isAdhocColumn(editedColumn)) {
+    return [editedColumn, undefined, undefined];
+  }
+  if (isSavedExpression(editedColumn)) {
+    return [undefined, editedColumn, undefined];
+  }
+  return [undefined, undefined, editedColumn];
+};
+
 const ColumnSelectPopover = ({
   columns,
   editedColumn,
   onChange,
   onClose,
+  setLabel,
+  getCurrentTab,
+  label,
+  isAdhocColumnsEnabled,
 }: ColumnSelectPopoverProps) => {
+  const [initialLabel] = useState(label);
   const [
+    initialAdhocColumn,
     initialCalculatedColumn,
     initialSimpleColumn,
-  ] = editedColumn?.expression
+  ] = getInitialColumnValues(editedColumn);
-    ? [editedColumn, undefined]
+
-    : [undefined, editedColumn];
+  const [adhocColumn, setAdhocColumn] = useState<AdhocColumn | undefined>(
-  const [selectedCalculatedColumn, setSelectedCalculatedColumn] = useState(
+    initialAdhocColumn,
-    initialCalculatedColumn,
-  );
-  const [selectedSimpleColumn, setSelectedSimpleColumn] = useState(
-    initialSimpleColumn,
   );
+  const [selectedCalculatedColumn, setSelectedCalculatedColumn] = useState<
+    ColumnMeta | undefined
+  >(initialCalculatedColumn);
+  const [selectedSimpleColumn, setSelectedSimpleColumn] = useState<
+    ColumnMeta | undefined
+  >(initialSimpleColumn);
+
+  const sqlEditorRef = useRef(null);
 
   const [calculatedColumns, simpleColumns] = useMemo(
     () =>
@@ -81,6 +122,15 @@ const ColumnSelectPopover = ({
     [columns],
   );
 
+  const onSqlExpressionChange = useCallback(
+    sqlExpression => {
+      setAdhocColumn({ label, sqlExpression } as AdhocColumn);
+      setSelectedSimpleColumn(undefined);
+      setSelectedCalculatedColumn(undefined);
+    },
+    [label],
+  );
+
   const onCalculatedColumnChange = useCallback(
     selectedColumnName => {
       const selectedColumn = calculatedColumns.find(
@@ -88,8 +138,12 @@ const ColumnSelectPopover = ({
       );
       setSelectedCalculatedColumn(selectedColumn);
       setSelectedSimpleColumn(undefined);
+      setAdhocColumn(undefined);
+      setLabel(
+        selectedColumn?.verbose_name || selectedColumn?.column_name || '',
+      );
     },
-    [calculatedColumns],
+    [calculatedColumns, setLabel],
   );
 
   const onSimpleColumnChange = useCallback(
@@ -99,33 +153,79 @@ const ColumnSelectPopover = ({
       );
       setSelectedCalculatedColumn(undefined);
       setSelectedSimpleColumn(selectedColumn);
+      setAdhocColumn(undefined);
+      setLabel(
+        selectedColumn?.verbose_name || selectedColumn?.column_name || '',
+      );
     },
-    [simpleColumns],
+    [setLabel, simpleColumns],
   );
 
-  const defaultActiveTabKey =
+  const defaultActiveTabKey = initialAdhocColumn
-    initialSimpleColumn || calculatedColumns.length === 0 ? 'simple' : 'saved';
+    ? 'sqlExpression'
+    : initialSimpleColumn || calculatedColumns.length === 0
+    ? 'simple'
+    : 'saved';
+
+  useEffect(() => {
+    getCurrentTab(defaultActiveTabKey);
+  }, [defaultActiveTabKey, getCurrentTab]);
 
   const onSave = useCallback(() => {
-    const selectedColumn = selectedCalculatedColumn || selectedSimpleColumn;
+    if (adhocColumn && adhocColumn.label !== label) {
+      adhocColumn.label = label;
+    }
+    const selectedColumn =
+      adhocColumn || selectedCalculatedColumn || selectedSimpleColumn;
     if (!selectedColumn) {
       return;
     }
     onChange(selectedColumn);
     onClose();
-  }, [onChange, onClose, selectedCalculatedColumn, selectedSimpleColumn]);
+  }, [
+    adhocColumn,
+    label,
+    onChange,
+    onClose,
+    selectedCalculatedColumn,
+    selectedSimpleColumn,
+  ]);
 
   const onResetStateAndClose = useCallback(() => {
     setSelectedCalculatedColumn(initialCalculatedColumn);
     setSelectedSimpleColumn(initialSimpleColumn);
+    setAdhocColumn(initialAdhocColumn);
     onClose();
-  }, [initialCalculatedColumn, initialSimpleColumn, onClose]);
+  }, [
+    initialAdhocColumn,
+    initialCalculatedColumn,
+    initialSimpleColumn,
+    onClose,
+  ]);
 
-  const stateIsValid = selectedCalculatedColumn || selectedSimpleColumn;
+  const onTabChange = useCallback(
+    tab => {
+      getCurrentTab(tab);
+      // @ts-ignore
+      sqlEditorRef.current?.editor.focus();
+    },
+    [getCurrentTab],
+  );
+
+  const onSqlEditorFocus = useCallback(() => {
+    // @ts-ignore
+    sqlEditorRef.current?.editor.resize();
+  }, []);
+
+  const stateIsValid =
+    adhocColumn || selectedCalculatedColumn || selectedSimpleColumn;
   const hasUnsavedChanges =
+    initialLabel !== label ||
     selectedCalculatedColumn?.column_name !==
       initialCalculatedColumn?.column_name ||
-    selectedSimpleColumn?.column_name !== initialSimpleColumn?.column_name;
+    selectedSimpleColumn?.column_name !== initialSimpleColumn?.column_name ||
+    adhocColumn?.sqlExpression !== initialAdhocColumn?.sqlExpression;
+
   const savedExpressionsLabel = t('Saved expressions');
   const simpleColumnsLabel = t('Column');
 
@@ -134,8 +234,12 @@ const ColumnSelectPopover = ({
       <Tabs
         id="adhoc-metric-edit-tabs"
         defaultActiveKey={defaultActiveTabKey}
+        onChange={onTabChange}
         className="adhoc-metric-edit-tabs"
         allowOverflow
+        css={css`
+          height: ${POPOVER_INITIAL_HEIGHT}px;
+        `}
       >
         <Tabs.TabPane key="saved" tab={t('Saved')}>
           <FormItem label={savedExpressionsLabel}>
@@ -178,6 +282,28 @@ const ColumnSelectPopover = ({
             />
           </FormItem>
         </Tabs.TabPane>
+        {isAdhocColumnsEnabled && (
+          <Tabs.TabPane key="sqlExpression" tab={t('Custom SQL')}>
+            <SQLEditor
+              value={
+                adhocColumn?.sqlExpression ||
+                selectedSimpleColumn?.column_name ||
+                selectedCalculatedColumn?.expression
+              }
+              onFocus={onSqlEditorFocus}
+              showLoadingForImport
+              onChange={onSqlExpressionChange}
+              width="100%"
+              height={`${POPOVER_INITIAL_HEIGHT - 80}px`}
+              showGutter={false}
+              editorProps={{ $blockScrolling: true }}
+              enableLiveAutocompletion
+              className="filter-sql-editor"
+              wrapEnabled
+              ref={sqlEditorRef}
+            />
+          </Tabs.TabPane>
+        )}
       </Tabs>
       <div>
         <Button buttonSize="small" onClick={onResetStateAndClose} cta>

superset-frontend/src/explore/components/controls/DndColumnSelectControl/ColumnSelectPopoverTrigger.tsx

@@ -16,16 +16,27 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import React, { useCallback, useMemo, useState } from 'react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
-import { ColumnMeta } from '@superset-ui/chart-controls';
+import {
+  AdhocColumn,
+  FeatureFlag,
+  isFeatureEnabled,
+  t,
+} from '@superset-ui/core';
+import {
+  ColumnMeta,
+  isAdhocColumn,
+  isColumnMeta,
+} from '@superset-ui/chart-controls';
 import Popover from 'src/components/Popover';
 import { ExplorePopoverContent } from 'src/explore/components/ExploreContentPopover';
 import ColumnSelectPopover from './ColumnSelectPopover';
+import { DndColumnSelectPopoverTitle } from './DndColumnSelectPopoverTitle';
 
 interface ColumnSelectPopoverTriggerProps {
   columns: ColumnMeta[];
-  editedColumn?: ColumnMeta;
+  editedColumn?: ColumnMeta | AdhocColumn;
-  onColumnEdit: (editedColumn: ColumnMeta) => void;
+  onColumnEdit: (editedColumn: ColumnMeta | AdhocColumn) => void;
   isControlledComponent?: boolean;
   visible?: boolean;
   togglePopover?: (visible: boolean) => void;
@@ -33,6 +44,11 @@ interface ColumnSelectPopoverTriggerProps {
   children: React.ReactNode;
 }
 
+const defaultPopoverLabel = t('My column');
+const editableTitleTab = 'sqlExpression';
+
+const isAdhocColumnsEnabled = isFeatureEnabled(FeatureFlag.UX_BETA);
+
 const ColumnSelectPopoverTrigger = ({
   columns,
   editedColumn,
@@ -41,7 +57,21 @@ const ColumnSelectPopoverTrigger = ({
   children,
   ...props
 }: ColumnSelectPopoverTriggerProps) => {
+  const [popoverLabel, setPopoverLabel] = useState(defaultPopoverLabel);
   const [popoverVisible, setPopoverVisible] = useState(false);
+  const [isTitleEditDisabled, setIsTitleEditDisabled] = useState(true);
+  const [hasCustomLabel, setHasCustomLabel] = useState(false);
+
+  let initialPopoverLabel = defaultPopoverLabel;
+  if (editedColumn && isColumnMeta(editedColumn)) {
+    initialPopoverLabel = editedColumn.verbose_name || editedColumn.column_name;
+  } else if (editedColumn && isAdhocColumn(editedColumn)) {
+    initialPopoverLabel = editedColumn.label || defaultPopoverLabel;
+  }
+
+  useEffect(() => {
+    setPopoverLabel(initialPopoverLabel);
+  }, [initialPopoverLabel, popoverVisible]);
 
   const togglePopover = useCallback((visible: boolean) => {
     setPopoverVisible(visible);
@@ -67,6 +97,10 @@ const ColumnSelectPopoverTrigger = ({
         handleClosePopover: closePopover,
       };
 
+  const getCurrentTab = useCallback((tab: string) => {
+    setIsTitleEditDisabled(tab !== editableTitleTab);
+  }, []);
+
   const overlayContent = useMemo(
     () => (
       <ExplorePopoverContent>
@@ -75,10 +109,38 @@ const ColumnSelectPopoverTrigger = ({
           columns={columns}
           onClose={handleClosePopover}
           onChange={onColumnEdit}
+          label={popoverLabel}
+          setLabel={setPopoverLabel}
+          getCurrentTab={getCurrentTab}
+          isAdhocColumnsEnabled={isAdhocColumnsEnabled}
         />
       </ExplorePopoverContent>
     ),
-    [columns, editedColumn, handleClosePopover, onColumnEdit],
+    [
+      columns,
+      editedColumn,
+      getCurrentTab,
+      handleClosePopover,
+      onColumnEdit,
+      popoverLabel,
+    ],
+  );
+
+  const onLabelChange = useCallback((e: any) => {
+    setPopoverLabel(e.target.value);
+    setHasCustomLabel(true);
+  }, []);
+
+  const popoverTitle = useMemo(
+    () => (
+      <DndColumnSelectPopoverTitle
+        title={popoverLabel}
+        onChange={onLabelChange}
+        isEditDisabled={isTitleEditDisabled}
+        hasCustomLabel={hasCustomLabel}
+      />
+    ),
+    [hasCustomLabel, isTitleEditDisabled, onLabelChange, popoverLabel],
   );
 
   return (
@@ -89,6 +151,7 @@ const ColumnSelectPopoverTrigger = ({
       defaultVisible={visible}
       visible={visible}
       onVisibleChange={handleTogglePopover}
+      title={isAdhocColumnsEnabled && popoverTitle}
       destroyTooltipOnHide
     >
       {children}

superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndColumnSelect.tsx

@@ -17,8 +17,14 @@
  * under the License.
  */
 import React, { useCallback, useMemo, useState } from 'react';
-import { FeatureFlag, isFeatureEnabled, tn } from '@superset-ui/core';
+import {
-import { ColumnMeta } from '@superset-ui/chart-controls';
+  AdhocColumn,
+  FeatureFlag,
+  isFeatureEnabled,
+  tn,
+  QueryFormColumn,
+} from '@superset-ui/core';
+import { ColumnMeta, isColumnMeta } from '@superset-ui/chart-controls';
 import { isEmpty } from 'lodash';
 import DndSelectLabel from 'src/explore/components/controls/DndColumnSelectControl/DndSelectLabel';
 import OptionWrapper from 'src/explore/components/controls/DndColumnSelectControl/OptionWrapper';
@@ -29,7 +35,7 @@ import { useComponentDidUpdate } from 'src/common/hooks/useComponentDidUpdate';
 import ColumnSelectPopoverTrigger from './ColumnSelectPopoverTrigger';
 import { DndControlProps } from './types';
 
-export type DndColumnSelectProps = DndControlProps<string> & {
+export type DndColumnSelectProps = DndControlProps<QueryFormColumn> & {
   options: Record<string, ColumnMeta>;
 };
 
@@ -123,7 +129,8 @@ export function DndColumnSelect(props: DndColumnSelectProps) {
       Object.values(options).filter(
         col =>
           !optionSelector.values
-            .map(val => val.column_name)
+            .filter(isColumnMeta)
+            .map((val: ColumnMeta) => val.column_name)
             .includes(col.column_name),
       ),
     [optionSelector.values, options],
@@ -136,7 +143,11 @@ export function DndColumnSelect(props: DndColumnSelectProps) {
           <ColumnSelectPopoverTrigger
             columns={popoverOptions}
             onColumnEdit={newColumn => {
-              optionSelector.replace(idx, newColumn.column_name);
+              if (isColumnMeta(newColumn)) {
+                optionSelector.replace(idx, newColumn.column_name);
+              } else {
+                optionSelector.replace(idx, newColumn as AdhocColumn);
+              }
               onChange(optionSelector.getValues());
             }}
             editedColumn={column}
@@ -177,8 +188,12 @@ export function DndColumnSelect(props: DndColumnSelectProps) {
   );
 
   const addNewColumnWithPopover = useCallback(
-    (newColumn: ColumnMeta) => {
+    (newColumn: ColumnMeta | AdhocColumn) => {
-      optionSelector.add(newColumn.column_name);
+      if (isColumnMeta(newColumn)) {
+        optionSelector.add(newColumn.column_name);
+      } else {
+        optionSelector.add(newColumn as AdhocColumn);
+      }
       onChange(optionSelector.getValues());
     },
     [onChange, optionSelector],

superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndColumnSelectPopoverTitle.jsx

@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import React, { useCallback, useState } from 'react';
+import { t } from '@superset-ui/core';
+import { Input } from 'src/common/components';
+import { Tooltip } from 'src/components/Tooltip';
+
+export const DndColumnSelectPopoverTitle = ({
+  title,
+  onChange,
+  isEditDisabled,
+  hasCustomLabel,
+}) => {
+  const [isHovered, setIsHovered] = useState(false);
+  const [isEditMode, setIsEditMode] = useState(false);
+
+  const onMouseOver = useCallback(() => {
+    setIsHovered(true);
+  }, []);
+
+  const onMouseOut = useCallback(() => {
+    setIsHovered(false);
+  }, []);
+
+  const onClick = useCallback(() => {
+    setIsEditMode(true);
+  }, []);
+
+  const onBlur = useCallback(() => {
+    setIsEditMode(false);
+  }, []);
+
+  const onInputBlur = useCallback(
+    e => {
+      if (e.target.value === '') {
+        onChange(e);
+      }
+      onBlur();
+    },
+    [onBlur, onChange],
+  );
+
+  const defaultLabel = t('My column');
+
+  if (isEditDisabled) {
+    return <span>{title || defaultLabel}</span>;
+  }
+
+  return isEditMode ? (
+    <Input
+      className="metric-edit-popover-label-input"
+      type="text"
+      placeholder={title}
+      value={hasCustomLabel ? title : ''}
+      autoFocus
+      onChange={onChange}
+      onBlur={onInputBlur}
+    />
+  ) : (
+    <Tooltip placement="top" title={t('Click to edit label')}>
+      <span
+        className="AdhocMetricEditPopoverTitle inline-editable"
+        data-test="AdhocMetricEditTitle#trigger"
+        onMouseOver={onMouseOver}
+        onMouseOut={onMouseOut}
+        onClick={onClick}
+        onBlur={onBlur}
+        role="button"
+        tabIndex={0}
+      >
+        {title || defaultLabel}
+        &nbsp;
+        <i
+          className="fa fa-pencil"
+          style={{ color: isHovered ? 'black' : 'grey' }}
+        />
+      </span>
+    </Tooltip>
+  );
+};

superset-frontend/src/explore/components/controls/DndColumnSelectControl/OptionWrapper.tsx

@@ -31,7 +31,7 @@ import {
 import { Tooltip } from 'src/components/Tooltip';
 import { StyledColumnOption } from 'src/explore/components/optionRenderers';
 import { styled } from '@superset-ui/core';
-import { ColumnMeta } from '@superset-ui/chart-controls';
+import { ColumnMeta, isAdhocColumn } from '@superset-ui/chart-controls';
 import Option from './Option';
 
 export const OptionLabel = styled.div`
@@ -135,14 +135,20 @@ export default function OptionWrapper(
     );
   };
 
-  const ColumnOption = () => (
+  const ColumnOption = () => {
-    <StyledColumnOption
+    const transformedCol =
-      column={column as ColumnMeta}
+      column && isAdhocColumn(column)
-      labelRef={labelRef}
+        ? { verbose_name: column.label, expression: column.sqlExpression }
-      showTooltip={!!shouldShowTooltip}
+        : column;
-      showType
+    return (
-    />
+      <StyledColumnOption
-  );
+        column={transformedCol as ColumnMeta}
+        labelRef={labelRef}
+        showTooltip={!!shouldShowTooltip}
+        showType
+      />
+    );
+  };
 
   const Label = () => {
     if (label) {

superset-frontend/src/explore/components/controls/DndColumnSelectControl/types.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 import { ReactNode } from 'react';
-import { JsonValue } from '@superset-ui/core';
+import { AdhocColumn, JsonValue } from '@superset-ui/core';
 import { ControlComponentProps } from 'src/explore/components/Control';
 import { ColumnMeta } from '@superset-ui/chart-controls';
 
@@ -26,7 +26,7 @@ export interface OptionProps {
   index: number;
   label?: string;
   tooltipTitle?: string;
-  column?: ColumnMeta;
+  column?: ColumnMeta | AdhocColumn;
   clickClose: (index: number) => void;
   withCaret?: boolean;
   isExtra?: boolean;

superset-frontend/src/explore/components/controls/DndColumnSelectControl/utils/optionSelector.ts

@@ -16,11 +16,25 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { ColumnMeta } from '@superset-ui/chart-controls';
+import { ColumnMeta, isColumnMeta } from '@superset-ui/chart-controls';
-import { ensureIsArray } from '@superset-ui/core';
+import {
+  AdhocColumn,
+  ensureIsArray,
+  QueryFormColumn,
+  isPhysicalColumn,
+} from '@superset-ui/core';
+
+const getColumnNameOrAdhocColumn = (
+  column: ColumnMeta | AdhocColumn,
+): QueryFormColumn => {
+  if (isColumnMeta(column)) {
+    return column.column_name;
+  }
+  return column as AdhocColumn;
+};
 
 export class OptionSelector {
-  values: ColumnMeta[];
+  values: (ColumnMeta | AdhocColumn)[];
 
   options: Record<string, ColumnMeta>;
 
@@ -29,23 +43,28 @@ export class OptionSelector {
   constructor(
     options: Record<string, ColumnMeta>,
     multi: boolean,
-    initialValues?: string[] | string | null,
+    initialValues?: QueryFormColumn[] | QueryFormColumn | null,
   ) {
     this.options = options;
     this.multi = multi;
     this.values = ensureIsArray(initialValues)
       .map(value => {
-        if (value && value in options) {
+        if (value && isPhysicalColumn(value) && value in options) {
           return options[value];
         }
+        if (!isPhysicalColumn(value)) {
+          return value;
+        }
         return null;
       })
       .filter(Boolean) as ColumnMeta[];
   }
 
-  add(value: string) {
+  add(value: QueryFormColumn) {
-    if (value in this.options) {
+    if (isPhysicalColumn(value) && value in this.options) {
       this.values.push(this.options[value]);
+    } else if (!isPhysicalColumn(value)) {
+      this.values.push(value as AdhocColumn);
     }
   }
 
@@ -53,9 +72,9 @@ export class OptionSelector {
     this.values.splice(idx, 1);
   }
 
-  replace(idx: number, value: string) {
+  replace(idx: number, value: QueryFormColumn) {
     if (this.values[idx]) {
-      this.values[idx] = this.options[value];
+      this.values[idx] = isPhysicalColumn(value) ? this.options[value] : value;
     }
   }
 
@@ -63,14 +82,27 @@ export class OptionSelector {
     [this.values[a], this.values[b]] = [this.values[b], this.values[a]];
   }
 
-  has(value: string): boolean {
+  has(value: QueryFormColumn): boolean {
-    return ensureIsArray(this.getValues()).includes(value);
+    return this.values.some(col => {
+      if (isPhysicalColumn(value)) {
+        return (
+          (col as ColumnMeta).column_name === value ||
+          (col as AdhocColumn).label === value
+        );
+      }
+      return (
+        (col as ColumnMeta).column_name === value.label ||
+        (col as AdhocColumn).label === value.label
+      );
+    });
   }
 
-  getValues(): string[] | string | undefined {
+  getValues(): QueryFormColumn[] | QueryFormColumn | undefined {
     if (!this.multi) {
-      return this.values.length > 0 ? this.values[0].column_name : undefined;
+      return this.values.length > 0
+        ? getColumnNameOrAdhocColumn(this.values[0])
+        : undefined;
     }
-    return this.values.map(option => option.column_name);
+    return this.values.map(getColumnNameOrAdhocColumn);
   }
 }

superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopover/index.jsx

@@ -31,6 +31,10 @@ import AdhocFilter, {
 import AdhocFilterEditPopoverSimpleTabContent from 'src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent';
 import AdhocFilterEditPopoverSqlTabContent from 'src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSqlTabContent';
 import columnType from 'src/explore/components/controls/FilterControl/columnType';
+import {
+  POPOVER_INITIAL_HEIGHT,
+  POPOVER_INITIAL_WIDTH,
+} from 'src/explore/constants';
 
 const propTypes = {
   adhocFilter: PropTypes.instanceOf(AdhocFilter).isRequired,
@@ -55,9 +59,6 @@ const ResizeIcon = styled.i`
   margin-left: ${({ theme }) => theme.gridUnit * 2}px;
 `;
 
-const startingWidth = 320;
-const startingHeight = 240;
-
 const FilterPopoverContentContainer = styled.div`
   .adhoc-filter-edit-tabs > .nav-tabs {
     margin-bottom: ${({ theme }) => theme.gridUnit * 2}px;
@@ -98,8 +99,8 @@ export default class AdhocFilterEditPopover extends React.Component {
 
     this.state = {
       adhocFilter: this.props.adhocFilter,
-      width: startingWidth,
+      width: POPOVER_INITIAL_WIDTH,
-      height: startingHeight,
+      height: POPOVER_INITIAL_HEIGHT,
       activeKey: this.props?.adhocFilter?.expressionType || 'SIMPLE',
     };
 
@@ -137,11 +138,11 @@ export default class AdhocFilterEditPopover extends React.Component {
     this.setState({
       width: Math.max(
         this.dragStartWidth + (e.clientX - this.dragStartX),
-        startingWidth,
+        POPOVER_INITIAL_WIDTH,
       ),
       height: Math.max(
         this.dragStartHeight + (e.clientY - this.dragStartY) * 2,
-        startingHeight,
+        POPOVER_INITIAL_HEIGHT,
       ),
     });
   }

superset-frontend/src/explore/components/controls/MetricControl/AdhocMetricEditPopover/index.jsx

@@ -30,7 +30,11 @@ import { SQLEditor } from 'src/components/AsyncAceEditor';
 import sqlKeywords from 'src/SqlLab/utils/sqlKeywords';
 import { noOp } from 'src/utils/common';
 
-import { AGGREGATES_OPTIONS } from 'src/explore/constants';
+import {
+  AGGREGATES_OPTIONS,
+  POPOVER_INITIAL_HEIGHT,
+  POPOVER_INITIAL_WIDTH,
+} from 'src/explore/constants';
 import columnType from 'src/explore/components/controls/MetricControl/columnType';
 import savedMetricType from 'src/explore/components/controls/MetricControl/savedMetricType';
 import AdhocMetric, {
@@ -73,9 +77,6 @@ const StyledSelect = styled(Select)`
 
 export const SAVED_TAB_KEY = 'SAVED';
 
-const startingWidth = 320;
-const startingHeight = 240;
-
 export default class AdhocMetricEditPopover extends React.PureComponent {
   // "Saved" is a default tab unless there are no saved metrics for dataset
   defaultActiveTabKey =
@@ -103,8 +104,8 @@ export default class AdhocMetricEditPopover extends React.PureComponent {
     this.state = {
       adhocMetric: this.props.adhocMetric,
       savedMetric: this.props.savedMetric,
-      width: startingWidth,
+      width: POPOVER_INITIAL_WIDTH,
-      height: startingHeight,
+      height: POPOVER_INITIAL_HEIGHT,
     };
 
     document.addEventListener('mouseup', this.onMouseUp);
@@ -225,11 +226,11 @@ export default class AdhocMetricEditPopover extends React.PureComponent {
     this.setState({
       width: Math.max(
         this.dragStartWidth + (e.clientX - this.dragStartX),
-        startingWidth,
+        POPOVER_INITIAL_WIDTH,
       ),
       height: Math.max(
         this.dragStartHeight + (e.clientY - this.dragStartY) * 2,
-        startingHeight,
+        POPOVER_INITIAL_HEIGHT,
       ),
     });
   }

superset-frontend/src/explore/constants.ts

@@ -151,3 +151,6 @@ export enum FILTER_BOX_MIGRATION_STATES {
 export const FILTER_BOX_TRANSITION_SNOOZED_AT =
   'filter_box_transition_snoozed_at';
 export const FILTER_BOX_TRANSITION_SNOOZE_DURATION = 24 * 60 * 60 * 1000; // 24 hours
+
+export const POPOVER_INITIAL_HEIGHT = 240;
+export const POPOVER_INITIAL_WIDTH = 320;

superset-frontend/src/filters/components/GroupBy/GroupByFilterPlugin.tsx

@@ -16,7 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { ensureIsArray, ExtraFormData, t, tn } from '@superset-ui/core';
+import {
+  ensureIsArray,
+  ExtraFormData,
+  getColumnLabel,
+  t,
+  tn,
+} from '@superset-ui/core';
 import React, { useEffect, useState } from 'react';
 import { FormItemProps } from 'antd/lib/form';
 import { Select } from 'src/components';
@@ -62,15 +68,11 @@ export default function PluginFilterGroupBy(props: PluginFilterGroupByProps) {
     // so we can process it like this `JSON.stringify` or start to use `Immer`
   }, [JSON.stringify(defaultValue), multiSelect]);
 
-  const groupby = formData?.groupby?.[0]?.length
+  const groupbys = ensureIsArray(formData.groupby).map(getColumnLabel);
-    ? formData?.groupby?.[0]
+  const groupby = groupbys[0].length ? groupbys[0] : null;
-    : null;
 
   const withData = groupby
-    ? data.filter(dataItem =>
+    ? data.filter(row => groupby.includes(row.column_name as string))
-        // @ts-ignore
-        groupby.includes(dataItem.column_name),
-      )
     : data;
 
   const columns = data ? withData : [];

superset-frontend/src/filters/components/Range/RangeFilterPlugin.tsx

@@ -17,6 +17,8 @@
  * under the License.
  */
 import {
+  ensureIsArray,
+  getColumnLabel,
   getNumberFormatter,
   NumberFormats,
   styled,
@@ -119,7 +121,7 @@ export default function RangeFilterPlugin(props: PluginFilterRangeProps) {
   // @ts-ignore
   const { min, max }: { min: number; max: number } = row;
   const { groupby, defaultValue, inputRef } = formData;
-  const [col = ''] = groupby || [];
+  const [col = ''] = ensureIsArray(groupby).map(getColumnLabel);
   const [value, setValue] = useState<[number, number]>(
     defaultValue ?? [min, max],
   );

superset-frontend/src/filters/components/Select/SelectFilterPlugin.tsx

@@ -24,6 +24,7 @@ import {
   ensureIsArray,
   ExtraFormData,
   GenericDataType,
+  getColumnLabel,
   JsonObject,
   smartDateDetailedFormatter,
   t,
@@ -94,7 +95,10 @@ export default function PluginFilterSelect(props: PluginFilterSelectProps) {
     defaultToFirstItem,
     searchAllOptions,
   } = formData;
-  const groupby = ensureIsArray<string>(formData.groupby);
+  const groupby = useMemo(
+    () => ensureIsArray(formData.groupby).map(getColumnLabel),
+    [formData.groupby],
+  );
   const [col] = groupby;
   const [initialColtypeMap] = useState(coltypeMap);
   const [dataMask, dispatchDataMask] = useImmerReducer(reducer, {

superset-frontend/src/filters/components/Select/buildQuery.ts

@@ -19,6 +19,8 @@
 import {
   buildQueryContext,
   GenericDataType,
+  getColumnLabel,
+  isPhysicalColumn,
   QueryObject,
   QueryObjectFilterClause,
   BuildQuery,
@@ -35,15 +37,16 @@ const buildQuery: BuildQuery<PluginFilterSelectQueryFormData> = (
     const { columns = [], filters = [] } = baseQueryObject;
     const extraFilters: QueryObjectFilterClause[] = [];
     if (search) {
-      columns.forEach(column => {
+      columns.filter(isPhysicalColumn).forEach(column => {
-        if (coltypeMap[column] === GenericDataType.STRING) {
+        const label = getColumnLabel(column);
+        if (coltypeMap[label] === GenericDataType.STRING) {
           extraFilters.push({
             col: column,
             op: 'ILIKE',
             val: `%${search}%`,
           });
         } else if (
-          coltypeMap[column] === GenericDataType.NUMERIC &&
+          coltypeMap[label] === GenericDataType.NUMERIC &&
           !Number.isNaN(Number(search))
         ) {
           // for numeric columns we apply a >= where clause

superset/charts/schemas.py

@@ -582,6 +582,7 @@ class ChartDataBoxplotOptionsSchema(ChartDataPostProcessingOperationOptionsSchem
         "references to datasource metrics (strings), or ad-hoc metrics"
         "which are defined only within the query object. See "
         "`ChartDataAdhocMetricSchema` for the structure of ad-hoc metrics.",
+        allow_none=True,
     )
 
     whisker_type = fields.String(
@@ -771,8 +772,11 @@ class ChartDataPostProcessingOperationSchema(Schema):
 
 
 class ChartDataFilterSchema(Schema):
-    col = fields.String(
+    col = fields.Raw(
-        description="The column to filter.", required=True, example="country"
+        description="The column to filter by. Can be either a string (physical or "
+        "saved expression) or an object (adhoc column)",
+        required=True,
+        example="country",
     )
     op = fields.String(  # pylint: disable=invalid-name
         description="The comparison operator.",
@@ -961,7 +965,7 @@ class ChartDataQueryObjectSchema(Schema):
         deprecated=True,
     )
     groupby = fields.List(
-        fields.String(),
+        fields.Raw(),
         description="Columns by which to group the query. "
         "This field is deprecated, use `columns` instead.",
         allow_none=True,
@@ -1012,7 +1016,7 @@ class ChartDataQueryObjectSchema(Schema):
         description="Is the `query_object` a timeseries.", allow_none=True,
     )
     series_columns = fields.List(
-        fields.String(),
+        fields.Raw(),
         description="Columns to use when limiting series count. "
         "All columns must be present in the `columns` property. "
         "Requires `series_limit` and `series_limit_metric` to be set.",
@@ -1062,7 +1066,7 @@ class ChartDataQueryObjectSchema(Schema):
         allow_none=True,
     )
     columns = fields.List(
-        fields.String(),
+        fields.Raw(),
         description="Columns which to select in the query.",
         allow_none=True,
     )

superset/common/query_actions.py

@@ -28,7 +28,9 @@ from superset.utils.core import (
     extract_column_dtype,
     extract_dataframe_dtypes,
     ExtraFiltersReasonType,
+    get_column_name,
     get_time_filter_status,
+    is_adhoc_column,
 )
 
 if TYPE_CHECKING:
@@ -114,14 +116,16 @@ def _get_full(
         datasource, query_obj.applied_time_extras
     )
     payload["applied_filters"] = [
-        {"column": col}
+        {"column": get_column_name(col)}
         for col in filter_columns
-        if col in columns or col in applied_template_filters
+        if is_adhoc_column(col) or col in columns or col in applied_template_filters
     ] + applied_time_columns
     payload["rejected_filters"] = [
         {"reason": ExtraFiltersReasonType.COL_NOT_IN_DATASOURCE, "column": col}
         for col in filter_columns
-        if col not in columns and col not in applied_template_filters
+        if not is_adhoc_column(col)
+        and col not in columns
+        and col not in applied_template_filters
     ] + rejected_time_columns
 
     if result_type == ChartDataResultType.RESULTS and status != QueryStatus.FAILED:

superset/common/query_context.py

@@ -46,6 +46,7 @@ from superset.utils.core import (
     DatasourceDict,
     DTTM_ALIAS,
     error_msg_from_exception,
+    get_column_names_from_columns,
     get_column_names_from_metrics,
     get_metric_names,
     normalize_dttm_col,
@@ -453,7 +454,7 @@ class QueryContext:
             try:
                 invalid_columns = [
                     col
-                    for col in query_obj.columns
+                    for col in get_column_names_from_columns(query_obj.columns)
                     + get_column_names_from_metrics(query_obj.metrics or [])
                     if col not in self.datasource.column_names and col != DTTM_ALIAS
                 ]

superset/common/query_object.py

@@ -27,13 +27,14 @@ from superset.common.chart_data import ChartDataResultType
 from superset.connectors.base.models import BaseDatasource
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.exceptions import QueryObjectValidationError
-from superset.typing import Metric, OrderBy
+from superset.typing import Column, Metric, OrderBy
 from superset.utils import pandas_postprocessing
 from superset.utils.core import (
     apply_max_row_limit,
     DatasourceDict,
     DTTM_ALIAS,
     find_duplicates,
+    get_column_names,
     get_metric_names,
     is_adhoc_metric,
     json_int_dttm_ser,
@@ -83,7 +84,7 @@ class QueryObject:  # pylint: disable=too-many-instance-attributes
     annotation_layers: List[Dict[str, Any]]
     applied_time_extras: Dict[str, str]
     apply_fetch_values_predicate: bool
-    columns: List[str]
+    columns: List[Column]
     datasource: Optional[BaseDatasource]
     extras: Dict[str, Any]
     filter: List[QueryObjectFilterClause]
@@ -93,19 +94,19 @@ class QueryObject:  # pylint: disable=too-many-instance-attributes
     inner_to_dttm: Optional[datetime]
     is_rowcount: bool
     is_timeseries: bool
+    metrics: Optional[List[Metric]]
     order_desc: bool
     orderby: List[OrderBy]
-    metrics: Optional[List[Metric]]
+    post_processing: List[Dict[str, Any]]
     result_type: Optional[ChartDataResultType]
     row_limit: int
     row_offset: int
-    series_columns: List[str]
+    series_columns: List[Column]
     series_limit: int
     series_limit_metric: Optional[Metric]
     time_offsets: List[str]
     time_shift: Optional[timedelta]
     to_dttm: Optional[datetime]
-    post_processing: List[Dict[str, Any]]
 
     def __init__(  # pylint: disable=too-many-arguments,too-many-locals
         self,
@@ -113,7 +114,7 @@ class QueryObject:  # pylint: disable=too-many-instance-attributes
         annotation_layers: Optional[List[Dict[str, Any]]] = None,
         applied_time_extras: Optional[Dict[str, str]] = None,
         apply_fetch_values_predicate: bool = False,
-        columns: Optional[List[str]] = None,
+        columns: Optional[List[Column]] = None,
         datasource: Optional[DatasourceDict] = None,
         extras: Optional[Dict[str, Any]] = None,
         filters: Optional[List[QueryObjectFilterClause]] = None,
@@ -127,7 +128,7 @@ class QueryObject:  # pylint: disable=too-many-instance-attributes
         result_type: Optional[ChartDataResultType] = None,
         row_limit: Optional[int] = None,
         row_offset: Optional[int] = None,
-        series_columns: Optional[List[str]] = None,
+        series_columns: Optional[List[Column]] = None,
         series_limit: int = 0,
         series_limit_metric: Optional[Metric] = None,
         time_range: Optional[str] = None,
@@ -266,9 +267,9 @@ class QueryObject:  # pylint: disable=too-many-instance-attributes
 
     @property
     def column_names(self) -> List[str]:
-        """Return column names (labels). Reserved for future adhoc calculated
+        """Return column names (labels). Gives priority to groupbys if both groupbys
-        columns."""
+        and metrics are non-empty, otherwise returns column labels."""
-        return self.columns
+        return get_column_names(self.columns)
 
     def validate(
         self, raise_exceptions: Optional[bool] = True

superset/connectors/base/models.py

@@ -324,7 +324,7 @@ class BaseDatasource(
             if query_context:
                 column_names.update(
                     [
-                        column
+                        utils.get_column_name(column)
                         for query in query_context.queries
                         for column in query.columns
                     ]

superset/connectors/sqla/models.py

@@ -88,10 +88,12 @@ from superset.models.annotations import Annotation
 from superset.models.core import Database
 from superset.models.helpers import AuditMixinNullable, CertificationMixin, QueryResult
 from superset.sql_parse import ParsedQuery
-from superset.typing import AdhocMetric, Metric, OrderBy, QueryObjectDict
+from superset.typing import AdhocColumn, AdhocMetric, Metric, OrderBy, QueryObjectDict
 from superset.utils import core as utils
 from superset.utils.core import (
     GenericDataType,
+    get_column_name,
+    is_adhoc_column,
     QueryObjectFilterClause,
     remove_duplicates,
 )
@@ -877,6 +879,26 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
 
         return self.make_sqla_column_compatible(sqla_metric, label)
 
+    def adhoc_column_to_sqla(
+        self,
+        col: AdhocColumn,
+        template_processor: Optional[BaseTemplateProcessor] = None,
+    ) -> ColumnElement:
+        """
+        Turn an adhoc column into a sqlalchemy column.
+
+        :param col: Adhoc column definition
+        :param template_processor: template_processor instance
+        :returns: The metric defined as a sqlalchemy column
+        :rtype: sqlalchemy.sql.column
+        """
+        label = utils.get_column_name(col)
+        expression = col["sqlExpression"]
+        if template_processor and expression:
+            expression = template_processor.process_template(expression)
+        sqla_metric = literal_column(expression)
+        return self.make_sqla_column_compatible(sqla_metric, label)
+
     def make_sqla_column_compatible(
         self, sqla_col: ColumnElement, label: Optional[str] = None
     ) -> ColumnElement:
@@ -951,14 +973,14 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
     def get_sqla_query(  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
         self,
         apply_fetch_values_predicate: bool = False,
-        columns: Optional[List[str]] = None,
+        columns: Optional[List[Column]] = None,
         extras: Optional[Dict[str, Any]] = None,
         filter: Optional[  # pylint: disable=redefined-builtin
             List[QueryObjectFilterClause]
         ] = None,
         from_dttm: Optional[datetime] = None,
         granularity: Optional[str] = None,
-        groupby: Optional[List[str]] = None,
+        groupby: Optional[List[Column]] = None,
         inner_from_dttm: Optional[datetime] = None,
         inner_to_dttm: Optional[datetime] = None,
         is_rowcount: bool = False,
@@ -967,7 +989,7 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
         orderby: Optional[List[OrderBy]] = None,
         order_desc: bool = True,
         to_dttm: Optional[datetime] = None,
-        series_columns: Optional[List[str]] = None,
+        series_columns: Optional[List[Column]] = None,
         series_limit: Optional[int] = None,
         series_limit_metric: Optional[Metric] = None,
         row_limit: Optional[int] = None,
@@ -995,7 +1017,9 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
             "table_columns": [col.column_name for col in self.columns],
             "filter": filter,
         }
-        series_columns = series_columns or []
+        columns = columns or []
+        groupby = groupby or []
+        series_column_names = utils.get_column_names(series_columns or [])
         # deprecated, to be removed in 2.0
         if is_timeseries and timeseries_limit:
             series_limit = timeseries_limit
@@ -1021,6 +1045,7 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
         columns_by_name: Dict[str, TableColumn] = {
             col.column_name: col for col in self.columns
         }
+
         metrics_by_name: Dict[str, SqlMetric] = {m.metric_name: m for m in self.metrics}
 
         if not granularity and is_timeseries:
@@ -1092,7 +1117,6 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
         groupby_series_columns = {}
 
         # filter out the pseudo column  __timestamp from columns
-        columns = columns or []
         columns = [col for col in columns if col != utils.DTTM_ALIAS]
         dttm_col = columns_by_name.get(granularity) if granularity else None
 
@@ -1100,22 +1124,27 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
             # dedup columns while preserving order
             columns = groupby or columns
             for selected in columns:
-                # if groupby field/expr equals granularity field/expr
+                if isinstance(selected, str):
-                if selected == granularity:
+                    # if groupby field/expr equals granularity field/expr
-                    sqla_col = columns_by_name[selected]
+                    if selected == granularity:
-                    outer = sqla_col.get_timestamp_expression(
+                        table_col = columns_by_name[selected]
-                        time_grain=time_grain,
+                        outer = table_col.get_timestamp_expression(
-                        label=selected,
+                            time_grain=time_grain,
-                        template_processor=template_processor,
+                            label=selected,
-                    )
+                            template_processor=template_processor,
-                # if groupby field equals a selected column
+                        )
-                elif selected in columns_by_name:
+                    # if groupby field equals a selected column
-                    outer = columns_by_name[selected].get_sqla_col()
+                    elif selected in columns_by_name:
+                        outer = columns_by_name[selected].get_sqla_col()
+                    else:
+                        outer = literal_column(f"({selected})")
+                        outer = self.make_sqla_column_compatible(outer, selected)
                 else:
-                    outer = literal_column(f"({selected})")
+                    outer = self.adhoc_column_to_sqla(
-                    outer = self.make_sqla_column_compatible(outer, selected)
+                        col=selected, template_processor=template_processor
+                    )
                 groupby_all_columns[outer.name] = outer
-                if not series_columns or outer.name in series_columns:
+                if not series_column_names or outer.name in series_column_names:
                     groupby_series_columns[outer.name] = outer
                 select_exprs.append(outer)
         elif columns:
@@ -1190,29 +1219,36 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
         for flt in filter:  # type: ignore
             if not all(flt.get(s) for s in ["col", "op"]):
                 continue
-            col = flt["col"]
+            flt_col = flt["col"]
             val = flt.get("val")
             op = flt["op"].upper()
-            col_obj = (
+            col_obj: Optional[TableColumn] = None
-                dttm_col
+            sqla_col: Optional[Column] = None
-                if col == utils.DTTM_ALIAS and is_timeseries and dttm_col
+            if flt_col == utils.DTTM_ALIAS and is_timeseries and dttm_col:
-                else columns_by_name.get(col)
+                col_obj = dttm_col
-            )
+            elif is_adhoc_column(flt_col):
+                sqla_col = self.adhoc_column_to_sqla(flt_col)
+            else:
+                col_obj = columns_by_name.get(flt_col)
             filter_grain = flt.get("grain")
 
             if is_feature_enabled("ENABLE_TEMPLATE_REMOVE_FILTERS"):
-                if col in removed_filters:
+                if get_column_name(flt_col) in removed_filters:
                     # Skip generating SQLA filter when the jinja template handles it.
                     continue
 
-            if col_obj:
+            if col_obj or sqla_col is not None:
-                if filter_grain:
+                if sqla_col is not None:
+                    pass
+                elif col_obj and filter_grain:
                     sqla_col = col_obj.get_timestamp_expression(
                         time_grain=filter_grain, template_processor=template_processor
                     )
-                else:
+                elif col_obj:
                     sqla_col = col_obj.get_sqla_col()
-                col_spec = db_engine_spec.get_column_spec(col_obj.type)
+                col_spec = db_engine_spec.get_column_spec(
+                    col_obj.type if col_obj else None
+                )
                 is_list_target = op in (
                     utils.FilterOperator.IN.value,
                     utils.FilterOperator.NOT_IN.value,
@@ -1348,6 +1384,7 @@ class SqlaTable(Model, BaseDatasource):  # pylint: disable=too-many-public-metho
                 inner_groupby_exprs = []
                 inner_select_exprs = []
                 for gby_name, gby_obj in groupby_series_columns.items():
+                    label = get_column_name(gby_name)
                     inner = self.make_sqla_column_compatible(gby_obj, gby_name + "__")
                     inner_groupby_exprs.append(inner)
                     inner_select_exprs.append(inner)

superset/typing.py

@@ -58,6 +58,13 @@ class AdhocMetric(TypedDict, total=False):
     aggregate: str
     column: Optional[AdhocMetricColumn]
     expressionType: Literal["SIMPLE", "SQL"]
+    hasCustomLabel: Optional[bool]
+    label: Optional[str]
+    sqlExpression: Optional[str]
+
+
+class AdhocColumn(TypedDict, total=False):
+    hasCustomLabel: Optional[bool]
     label: Optional[str]
     sqlExpression: Optional[str]
 
@@ -72,6 +79,7 @@ FilterValue = Union[bool, datetime, float, int, str]
 FilterValues = Union[FilterValue, List[FilterValue], Tuple[FilterValue]]
 FormData = Dict[str, Any]
 Granularity = Union[str, Dict[str, Union[str, float]]]
+Column = Union[AdhocColumn, str]
 Metric = Union[AdhocMetric, str]
 OrderBy = Tuple[Metric, bool]
 QueryObjectDict = Dict[str, Any]

superset/utils/core.py

@@ -98,8 +98,10 @@ from superset.exceptions import (
     SupersetTimeoutException,
 )
 from superset.typing import (
+    AdhocColumn,
     AdhocMetric,
     AdhocMetricColumn,
+    Column,
     FilterValues,
     FlaskResponse,
     FormData,
@@ -1265,6 +1267,29 @@ def is_adhoc_metric(metric: Metric) -> TypeGuard[AdhocMetric]:
     return isinstance(metric, dict) and "expressionType" in metric
 
 
+def is_adhoc_column(column: Column) -> TypeGuard[AdhocColumn]:
+    return isinstance(column, dict)
+
+
+def get_column_name(column: Column) -> str:
+    """
+    Extract label from column
+
+    :param column: object to extract label from
+    :return: String representation of column
+    :raises ValueError: if metric object is invalid
+    """
+    if isinstance(column, dict):
+        label = column.get("label")
+        if label:
+            return label
+        expr = column.get("sqlExpression")
+        if expr:
+            return expr
+        raise Exception("Missing label")
+    return column
+
+
 def get_metric_name(metric: Metric) -> str:
     """
     Extract label from metric
@@ -1294,11 +1319,15 @@ def get_metric_name(metric: Metric) -> str:
     return metric  # type: ignore
 
 
-def get_metric_names(metrics: Sequence[Metric]) -> List[str]:
+def get_column_names(columns: Optional[Sequence[Column]]) -> List[str]:
-    return [metric for metric in map(get_metric_name, metrics) if metric]
+    return [column for column in map(get_column_name, columns or []) if column]
 
 
-def get_first_metric_name(metrics: Sequence[Metric]) -> Optional[str]:
+def get_metric_names(metrics: Optional[Sequence[Metric]]) -> List[str]:
+    return [metric for metric in map(get_metric_name, metrics or []) if metric]
+
+
+def get_first_metric_name(metrics: Optional[Sequence[Metric]]) -> Optional[str]:
     metric_labels = get_metric_names(metrics)
     return metric_labels[0] if metric_labels else None
 
@@ -1518,6 +1547,30 @@ def get_form_data_token(form_data: Dict[str, Any]) -> str:
     return form_data.get("token") or "token_" + uuid.uuid4().hex[:8]
 
 
+def get_column_name_from_column(column: Column) -> Optional[str]:
+    """
+    Extract the physical column that a column is referencing. If the column is
+    an adhoc column, always returns `None`.
+
+    :param column: Physical and ad-hoc column
+    :return: column name if physical column, otherwise None
+    """
+    if is_adhoc_column(column):
+        return None
+    return column  # type: ignore
+
+
+def get_column_names_from_columns(columns: List[Column]) -> List[str]:
+    """
+    Extract the physical columns that a list of columns are referencing. Ignore
+    adhoc columns
+
+    :param columns: Physical and adhoc columns
+    :return: column names of all physical columns
+    """
+    return [col for col in map(get_column_name_from_column, columns) if col]
+
+
 def get_column_name_from_metric(metric: Metric) -> Optional[str]:
     """
     Extract the column that a metric is referencing. If the metric isn't

superset/viz.py

@@ -66,13 +66,18 @@ from superset.exceptions import (
 )
 from superset.extensions import cache_manager, security_manager
 from superset.models.helpers import QueryResult
-from superset.typing import Metric, QueryObjectDict, VizData, VizPayload
+from superset.typing import Column, Metric, QueryObjectDict, VizData, VizPayload
 from superset.utils import core as utils, csv
 from superset.utils.cache import set_and_log_cache
 from superset.utils.core import (
     apply_max_row_limit,
     DTTM_ALIAS,
     ExtraFiltersReasonType,
+    get_column_name,
+    get_column_names,
+    get_column_names_from_columns,
+    get_metric_names,
+    is_adhoc_column,
     JS_MAX_INTEGER,
     merge_extra_filters,
     QueryMode,
@@ -132,7 +137,7 @@ class BaseViz:  # pylint: disable=too-many-public-methods
         self.query = ""
         self.token = utils.get_form_data_token(form_data)
 
-        self.groupby: List[str] = self.form_data.get("groupby") or []
+        self.groupby: List[Column] = self.form_data.get("groupby") or []
         self.time_shift = timedelta()
 
         self.status: Optional[str] = None
@@ -303,19 +308,30 @@ class BaseViz:  # pylint: disable=too-many-public-methods
         merge_extra_filters(self.form_data)
         utils.split_adhoc_filters_into_base_filters(self.form_data)
 
+    @staticmethod
+    def dedup_columns(*columns_args: Optional[List[Column]]) -> List[Column]:
+        # dedup groupby and columns while preserving order
+        labels: List[str] = []
+        deduped_columns: List[Column] = []
+        for columns in columns_args:
+            for column in columns or []:
+                label = get_column_name(column)
+                if label not in labels:
+                    deduped_columns.append(column)
+        return deduped_columns
+
     def query_obj(self) -> QueryObjectDict:  # pylint: disable=too-many-locals
         """Building a query object"""
         self.process_query_filters()
 
-        gb = self.groupby
         metrics = self.all_metrics or []
-        columns = self.form_data.get("columns") or []
+
-        # merge list and dedup while preserving order
+        groupby = self.dedup_columns(self.groupby, self.form_data.get("columns"))
-        groupby = list(OrderedDict.fromkeys(gb + columns))
+        groupby_labels = get_column_names(groupby)
 
         is_timeseries = self.is_timeseries
-        if DTTM_ALIAS in groupby:
+        if DTTM_ALIAS in groupby_labels:
-            groupby.remove(DTTM_ALIAS)
+            del groupby[groupby_labels.index(DTTM_ALIAS)]
             is_timeseries = True
 
         granularity = self.form_data.get("granularity") or self.form_data.get(
@@ -462,14 +478,16 @@ class BaseViz:  # pylint: disable=too-many-public-methods
             self.datasource, applied_time_extras
         )
         payload["applied_filters"] = [
-            {"column": col}
+            {"column": get_column_name(col)}
             for col in filter_columns
-            if col in columns or col in applied_template_filters
+            if is_adhoc_column(col) or col in columns or col in applied_template_filters
         ] + applied_time_columns
         payload["rejected_filters"] = [
             {"reason": ExtraFiltersReasonType.COL_NOT_IN_DATASOURCE, "column": col}
             for col in filter_columns
-            if col not in columns and col not in applied_template_filters
+            if not is_adhoc_column(col)
+            and col not in columns
+            and col not in applied_template_filters
         ] + rejected_time_columns
         if df is not None:
             payload["colnames"] = list(df.columns)
@@ -519,10 +537,12 @@ class BaseViz:  # pylint: disable=too-many-public-methods
             try:
                 invalid_columns = [
                     col
-                    for col in (query_obj.get("columns") or [])
+                    for col in get_column_names_from_columns(
-                    + (query_obj.get("groupby") or [])
+                        query_obj.get("columns") or []
+                    )
+                    + get_column_names_from_columns(query_obj.get("groupby") or [])
                     + utils.get_column_names_from_metrics(
-                        cast(List[Metric], query_obj.get("metrics") or [],)
+                        cast(List[Metric], query_obj.get("metrics") or [])
                     )
                     if col not in self.datasource.column_names
                 ]
@@ -688,16 +708,16 @@ class TableViz(BaseViz):
             else QueryMode.AGGREGATE
         )
 
-        columns: List[str] = []  # output columns sans time and percent_metric column
+        columns: List[str]  # output columns sans time and percent_metric column
         percent_columns: List[str] = []  # percent columns that needs extra computation
 
         if self.query_mode == QueryMode.RAW:
-            columns = utils.get_metric_names(self.form_data.get("all_columns") or [])
+            columns = get_metric_names(self.form_data.get("all_columns"))
         else:
-            columns = utils.get_metric_names(
+            columns = get_column_names(self.groupby) + get_metric_names(
-                self.groupby + (self.form_data.get("metrics") or [])
+                self.form_data.get("metrics")
             )
-            percent_columns = utils.get_metric_names(
+            percent_columns = get_metric_names(
                 self.form_data.get("percent_metrics") or []
             )
 
@@ -820,7 +840,7 @@ class TimeTableViz(BaseViz):
         values: Union[List[str], str] = self.metric_labels
         if self.form_data.get("groupby"):
             values = self.metric_labels[0]
-            columns = self.form_data.get("groupby")
+            columns = get_column_names(self.form_data.get("groupby"))
         pt = df.pivot_table(index=DTTM_ALIAS, columns=columns, values=values)
         pt.index = pt.index.map(str)
         pt = pt.sort_index()
@@ -866,7 +886,9 @@ class PivotTableViz(BaseViz):
             )
         if not metrics:
             raise QueryObjectValidationError(_("Please choose at least one metric"))
-        if set(groupby) & set(columns):
+        deduped_cols = self.dedup_columns(groupby, columns)
+
+        if len(deduped_cols) < (len(groupby) + len(columns)):
             raise QueryObjectValidationError(_("Group By' and 'Columns' can't overlap"))
         sort_by = self.form_data.get("timeseries_limit_metric")
         if sort_by:
@@ -931,18 +953,22 @@ class PivotTableViz(BaseViz):
         groupby = self.form_data.get("groupby") or []
         columns = self.form_data.get("columns") or []
 
-        for column_name in groupby + columns:
+        for column in groupby + columns:
-            column = self.datasource.get_column(column_name)
+            if is_adhoc_column(column):
-            if column and column.is_temporal:
+                # TODO: check data type
-                ts = df[column_name].apply(self._format_datetime)
+                pass
-                df[column_name] = ts
+            else:
+                column_obj = self.datasource.get_column(column)
+                if column_obj and column_obj.is_temporal:
+                    ts = df[column].apply(self._format_datetime)
+                    df[column] = ts
 
         if self.form_data.get("transpose_pivot"):
             groupby, columns = columns, groupby
 
         df = df.pivot_table(
-            index=groupby,
+            index=get_column_names(groupby),
-            columns=columns,
+            columns=get_column_names(columns),
             values=metrics,
             aggfunc=aggfuncs,
             margins=self.form_data.get("pivot_margins"),
@@ -1003,7 +1029,7 @@ class TreemapViz(BaseViz):
         if df.empty:
             return None
 
-        df = df.set_index(self.form_data.get("groupby"))
+        df = df.set_index(get_column_names(self.form_data.get("groupby")))
         chart_data = [
             {"name": metric, "children": self._nest(metric, df)}
             for metric in df.columns
@@ -1117,7 +1143,7 @@ class BubbleViz(NVD3Viz):
             query_obj["groupby"].append(self.form_data.get("series"))
 
         # dedup groupby if it happens to be the same
-        query_obj["groupby"] = list(dict.fromkeys(query_obj["groupby"]))
+        query_obj["groupby"] = self.dedup_columns(query_obj["groupby"])
 
         # pylint: disable=attribute-defined-outside-init
         self.x_metric = self.form_data["x"]
@@ -1142,7 +1168,7 @@ class BubbleViz(NVD3Viz):
         df["y"] = df[[utils.get_metric_name(self.y_metric)]]
         df["size"] = df[[utils.get_metric_name(self.z_metric)]]
         df["shape"] = "circle"
-        df["group"] = df[[self.series]]
+        df["group"] = df[[get_column_name(self.series)]]  # type: ignore
 
         series: Dict[Any, List[Any]] = defaultdict(list)
         for row in df.to_dict(orient="records"):
@@ -1335,7 +1361,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
         if aggregate:
             df = df.pivot_table(
                 index=DTTM_ALIAS,
-                columns=self.form_data.get("groupby"),
+                columns=get_column_names(self.form_data.get("groupby")),
                 values=self.metric_labels,
                 fill_value=0,
                 aggfunc=sum,
@@ -1343,7 +1369,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
         else:
             df = df.pivot_table(
                 index=DTTM_ALIAS,
-                columns=self.form_data.get("groupby"),
+                columns=get_column_names(self.form_data.get("groupby")),
                 values=self.metric_labels,
                 fill_value=self.pivot_fill_value,
             )
@@ -1716,15 +1742,15 @@ class HistogramViz(BaseViz):
 
         chart_data = []
         if len(self.groupby) > 0:
-            groups = df.groupby(self.groupby)
+            groups = df.groupby(get_column_names(self.groupby))
         else:
             groups = [((), df)]
         for keys, data in groups:
             chart_data.extend(
                 [
                     {
-                        "key": self.labelify(keys, column),
+                        "key": self.labelify(keys, get_column_name(column)),
-                        "values": data[column].tolist(),
+                        "values": data[get_column_name(column)].tolist(),
                     }
                     for column in self.columns
                 ]
@@ -1775,21 +1801,22 @@ class DistributionBarViz(BaseViz):
             return None
 
         metrics = self.metric_labels
-        columns = self.form_data.get("columns") or []
+        columns = get_column_names(self.form_data.get("columns"))
+        groupby = get_column_names(self.groupby)
 
         # pandas will throw away nulls when grouping/pivoting,
         # so we substitute NULL_STRING for any nulls in the necessary columns
-        filled_cols = self.groupby + columns
+        filled_cols = groupby + columns
         df = df.copy()
         df[filled_cols] = df[filled_cols].fillna(value=NULL_STRING)
 
         sortby = utils.get_metric_name(
             self.form_data.get("timeseries_limit_metric") or metrics[0]
         )
-        row = df.groupby(self.groupby).sum()[sortby].copy()
+        row = df.groupby(groupby).sum()[sortby].copy()
         is_asc = not self.form_data.get("order_desc")
         row.sort_values(ascending=is_asc, inplace=True)
-        pt = df.pivot_table(index=self.groupby, columns=columns, values=metrics)
+        pt = df.pivot_table(index=groupby, columns=columns, values=metrics)
         if self.form_data.get("contribution"):
             pt = pt.T
             pt = (pt / pt.sum()).T
@@ -1799,7 +1826,7 @@ class DistributionBarViz(BaseViz):
         pt = pt[metrics]
         chart_data = []
         for name, ys in pt.items():
-            if pt[name].dtype.kind not in "biufc" or name in self.groupby:
+            if pt[name].dtype.kind not in "biufc" or name in groupby:
                 continue
             if isinstance(name, str):
                 series_title = name
@@ -1834,7 +1861,7 @@ class SunburstViz(BaseViz):
         if df.empty:
             return None
         form_data = copy.deepcopy(self.form_data)
-        cols = form_data.get("groupby") or []
+        cols = get_column_names(form_data.get("groupby"))
         cols.extend(["m1", "m2"])
         metric = utils.get_metric_name(form_data["metric"])
         secondary_metric = (
@@ -1888,7 +1915,7 @@ class SankeyViz(BaseViz):
     def get_data(self, df: pd.DataFrame) -> VizData:
         if df.empty:
             return None
-        source, target = self.groupby
+        source, target = get_column_names(self.groupby)
         (value,) = self.metric_labels
         df.rename(
             columns={source: "source", target: "target", value: "value",}, inplace=True,
@@ -1995,7 +2022,7 @@ class CountryMapViz(BaseViz):
     def get_data(self, df: pd.DataFrame) -> VizData:
         if df.empty:
             return None
-        cols = [self.form_data.get("entity")]
+        cols = get_column_names([self.form_data.get("entity")])  # type: ignore
         metric = self.metric_labels[0]
         cols += [metric]
         ndf = df[cols]
@@ -2027,7 +2054,7 @@ class WorldMapViz(BaseViz):
         # pylint: disable=import-outside-toplevel
         from superset.examples import countries
 
-        cols = [self.form_data.get("entity")]
+        cols = get_column_names([self.form_data.get("entity")])  # type: ignore
         metric = utils.get_metric_name(self.form_data["metric"])
         secondary_metric = (
             utils.get_metric_name(self.form_data["secondary_metric"])
@@ -2197,8 +2224,8 @@ class HeatmapViz(BaseViz):
         if df.empty:
             return None
 
-        x = self.form_data.get("all_columns_x")
+        x = get_column_name(self.form_data.get("all_columns_x"))  # type: ignore
-        y = self.form_data.get("all_columns_y")
+        y = get_column_name(self.form_data.get("all_columns_y"))  # type: ignore
         v = self.metric_labels[0]
         if x == y:
             df.columns = ["x", "y", "v"]
@@ -2814,7 +2841,7 @@ class DeckGeoJson(BaseDeckGLViz):
         return query_obj
 
     def get_properties(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        geojson = data[self.form_data["geojson"]]
+        geojson = data[get_column_name(self.form_data["geojson"])]
         return json.loads(geojson)
 
 
@@ -2922,7 +2949,7 @@ class PairedTTestViz(BaseViz):
         if df.empty:
             return None
 
-        groups = self.form_data.get("groupby")
+        groups = get_column_names(self.form_data.get("groupby"))
         metrics = self.metric_labels
         df = df.pivot_table(index=DTTM_ALIAS, columns=groups, values=metrics)
         cols = []
@@ -3151,7 +3178,7 @@ class PartitionViz(NVD3TimeSeriesViz):
     def get_data(self, df: pd.DataFrame) -> VizData:
         if df.empty:
             return None
-        groups = self.form_data.get("groupby", [])
+        groups = get_column_names(self.form_data.get("groupby"))
         time_op = self.form_data.get("time_series_option", "not_time")
         if not groups:
             raise ValueError("Please choose at least one groupby")

tests/integration_tests/charts/api_tests.py

@@ -38,7 +38,6 @@ from superset.utils.core import get_example_default_schema
 
 from tests.integration_tests.base_api_tests import ApiOwnersTestCaseMixin
 from tests.integration_tests.base_tests import SupersetTestCase
-from tests.integration_tests.insert_chart_mixin import InsertChartMixin
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
 )
@@ -58,6 +57,7 @@ from tests.integration_tests.fixtures.unicode_dashboard import (
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
 )
+from tests.integration_tests.insert_chart_mixin import InsertChartMixin
 from tests.integration_tests.test_app import app
 from tests.integration_tests.utils.get_dashboards import get_dashboards_ids
 

tests/integration_tests/charts/data/api_tests.py

@@ -43,6 +43,7 @@ from superset.errors import SupersetErrorType
 from superset.extensions import async_query_manager, db
 from superset.models.annotations import AnnotationLayer
 from superset.models.slice import Slice
+from superset.typing import AdhocColumn
 from superset.utils.core import (
     AnnotationType,
     get_example_database,
@@ -61,6 +62,12 @@ from tests.integration_tests.fixtures.query_context import (
 
 CHART_DATA_URI = "api/v1/chart/data"
 CHARTS_FIXTURE_COUNT = 10
+ADHOC_COLUMN_FIXTURE: AdhocColumn = {
+    "hasCustomLabel": True,
+    "label": "male_or_female",
+    "sqlExpression": "case when gender = 'boy' then 'male' "
+    "when gender = 'girl' then 'female' else 'other' end",
+}
 
 
 class BaseTestChartDataApi(SupersetTestCase):
@@ -820,3 +827,23 @@ class TestGetChartDataApi(BaseTestChartDataApi):
         )
 
         self.assertEqual(rv.status_code, 404)
+
+    @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
+    def test_chart_data_with_adhoc_column(self):
+        """
+        Chart data API: Test query with adhoc column in both select and where clause
+        """
+        self.login(username="admin")
+        request_payload = get_query_context("birth_names")
+        request_payload["queries"][0]["columns"] = [ADHOC_COLUMN_FIXTURE]
+        request_payload["queries"][0]["filters"] = [
+            {"col": ADHOC_COLUMN_FIXTURE, "op": "IN", "val": ["male", "female"]}
+        ]
+        rv = self.post_assert_metric(CHART_DATA_URI, request_payload, "data")
+        response_payload = json.loads(rv.data.decode("utf-8"))
+        result = response_payload["result"][0]
+        data = result["data"]
+        assert {column for column in data[0].keys()} == {"male_or_female", "sum__num"}
+        unique_genders = {row["male_or_female"] for row in data}
+        assert unique_genders == {"male", "female"}
+        assert result["applied_filters"] == [{"column": "male_or_female"}]