diff --git a/docs/installation.rst b/docs/installation.rst index 295a20d09c..7224cd2f8c 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -559,6 +559,18 @@ The following keys in `superset_config.py` can be specified to configure CORS: * ``CORS_OPTIONS``: options passed to Flask-CORS (`documentation `) +DOMAIN SHARDING +--------------- + +Chrome allows up to 6 open connections per domain at a time. When there are more +than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for +next available socket. PR (`#5039 `) adds domain sharding to Superset, +and this feature will be enabled by configuration only (by default Superset +doesn't allow cross-domain request). + +*``SUPERSET_WEBSERVER_DOMAINS``: list of allowed hostnames for domain sharding feature. default `None` + + MIDDLEWARE ---------- diff --git a/superset/assets/spec/javascripts/explore/utils_spec.jsx b/superset/assets/spec/javascripts/explore/utils_spec.jsx index 633f7ce42c..323655dc06 100644 --- a/superset/assets/spec/javascripts/explore/utils_spec.jsx +++ b/superset/assets/spec/javascripts/explore/utils_spec.jsx @@ -1,5 +1,8 @@ +import sinon from 'sinon'; + import URI from 'urijs'; import { getExploreUrlAndPayload, getExploreLongUrl } from '../../../src/explore/exploreUtils'; +import * as hostNamesConfig from '../../../src/utils/hostNamesConfig'; describe('exploreUtils', () => { const location = window.location; @@ -128,6 +131,71 @@ describe('exploreUtils', () => { }); }); + describe('domain sharding', () => { + let stub; + const availableDomains = [ + 'http://localhost/', + 'domain1.com', 'domain2.com', 'domain3.com', + ]; + beforeEach(() => { + stub = sinon.stub(hostNamesConfig, 'availableDomains').value(availableDomains); + }); + afterEach(() => { + stub.restore(); + }); + + it('generate url to different domains', () => { + let url = getExploreUrlAndPayload({ + formData, + endpointType: 'json', + allowDomainSharding: true, + }).url; + expect(url).toMatch(availableDomains[0]); + + url = getExploreUrlAndPayload({ + formData, + endpointType: 'json', + allowDomainSharding: true, + }).url; + expect(url).toMatch(availableDomains[1]); + + url = getExploreUrlAndPayload({ + formData, + endpointType: 'json', + allowDomainSharding: true, + }).url; + expect(url).toMatch(availableDomains[2]); + + url = getExploreUrlAndPayload({ + formData, + endpointType: 'json', + allowDomainSharding: true, + }).url; + expect(url).toMatch(availableDomains[3]); + + // circle back to first available domain + url = getExploreUrlAndPayload({ + formData, + endpointType: 'json', + allowDomainSharding: true, + }).url; + expect(url).toMatch(availableDomains[0]); + }); + it('not generate url to different domains without flag', () => { + let csvURL = getExploreUrlAndPayload({ + formData, + endpointType: 'csv', + }).url; + expect(csvURL).toMatch(availableDomains[0]); + + csvURL = getExploreUrlAndPayload({ + formData, + endpointType: 'csv', + }).url; + expect(csvURL).toMatch(availableDomains[0]); + }); + }); + describe('getExploreLongUrl', () => { it('generates proper base url with form_data', () => { compareURI( diff --git a/superset/assets/src/chart/chartAction.js b/superset/assets/src/chart/chartAction.js index b816d87017..6c6ddab2fb 100644 --- a/superset/assets/src/chart/chartAction.js +++ b/superset/assets/src/chart/chartAction.js @@ -8,6 +8,7 @@ import { requiresQuery, ANNOTATION_SOURCE_TYPES } from '../modules/AnnotationTyp import { addDangerToast } from '../messageToasts/actions'; import { Logger, LOG_ACTIONS_LOAD_CHART } from '../logger'; import getClientErrorObject from '../utils/getClientErrorObject'; +import { allowCrossDomain } from '../utils/hostNamesConfig'; export const CHART_UPDATE_STARTED = 'CHART_UPDATE_STARTED'; export function chartUpdateStarted(queryController, latestQueryFormData, key) { @@ -145,6 +146,7 @@ export function runQuery(formData, force = false, timeout = 60, key) { formData, endpointType: 'json', force, + allowDomainSharding: true, }); const logStart = Logger.getTimestamp(); const controller = new AbortController(); @@ -152,12 +154,20 @@ export function runQuery(formData, force = false, timeout = 60, key) { dispatch(chartUpdateStarted(controller, payload, key)); - const queryPromise = SupersetClient.post({ + let querySettings = { url, postPayload: { form_data: payload }, signal, timeout: timeout * 1000, - }) + }; + if (allowCrossDomain) { + querySettings = { + ...querySettings, + mode: 'cors', + credentials: 'include', + }; + } + const queryPromise = SupersetClient.post(querySettings) .then(({ json }) => { Logger.append(LOG_ACTIONS_LOAD_CHART, { slice_id: key, diff --git a/superset/assets/src/explore/exploreUtils.js b/superset/assets/src/explore/exploreUtils.js index dcf562d13b..4eac3abce4 100644 --- a/superset/assets/src/explore/exploreUtils.js +++ b/superset/assets/src/explore/exploreUtils.js @@ -1,11 +1,23 @@ /* eslint camelcase: 0 */ import URI from 'urijs'; +import { availableDomains } from '../utils/hostNamesConfig'; export function getChartKey(explore) { const slice = explore.slice; return slice ? (slice.slice_id) : 0; } +let requestCounter = 0; +function getHostName(allowDomainSharding = false) { + let currentIndex = 0; + if (allowDomainSharding) { + currentIndex = requestCounter % availableDomains.length; + requestCounter += 1; + } + + return availableDomains[currentIndex]; +} + export function getAnnotationJsonUrl(slice_id, form_data, isNative) { if (slice_id === null || slice_id === undefined) { return null; @@ -49,6 +61,7 @@ export function getExploreUrlAndPayload({ force = false, curUrl = null, requestParams = {}, + allowDomainSharding = false, }) { if (!formData.datasource) { return null; @@ -57,7 +70,13 @@ export function getExploreUrlAndPayload({ // The search params from the window.location are carried through, // but can be specified with curUrl (used for unit tests to spoof // the window.location). - let uri = new URI([location.protocol, '//', location.host].join('')); + let uri = new URI({ + protocol: location.protocol.slice(0, -1), + hostname: getHostName(allowDomainSharding), + port: location.port ? location.port : '', + path: '/', + }); + if (curUrl) { uri = URI(URI(curUrl).search()); } @@ -105,7 +124,11 @@ export function getExploreUrlAndPayload({ } export function exportChart(formData, endpointType) { - const { url, payload } = getExploreUrlAndPayload({ formData, endpointType }); + const { url, payload } = getExploreUrlAndPayload({ + formData, + endpointType, + allowDomainSharding: false, + }); const exploreForm = document.createElement('form'); exploreForm.action = url; diff --git a/superset/assets/src/utils/hostNamesConfig.js b/superset/assets/src/utils/hostNamesConfig.js new file mode 100644 index 0000000000..8f381bb770 --- /dev/null +++ b/superset/assets/src/utils/hostNamesConfig.js @@ -0,0 +1,23 @@ +function getDomainsConfig() { + const appContainer = document.getElementById('app'); + if (!appContainer) { + return []; + } + + const bootstrapData = JSON.parse(appContainer.getAttribute('data-bootstrap')); + const availableDomains = new Set([location.hostname]); + if (bootstrapData && + bootstrapData.common && + bootstrapData.common.conf && + bootstrapData.common.conf.SUPERSET_WEBSERVER_DOMAINS + ) { + bootstrapData.common.conf.SUPERSET_WEBSERVER_DOMAINS.forEach((hostName) => { + availableDomains.add(hostName); + }); + } + return Array.from(availableDomains); +} + +export const availableDomains = getDomainsConfig(); + +export const allowCrossDomain = availableDomains.length > 1; diff --git a/superset/config.py b/superset/config.py index a34e5d96f4..a9622db739 100644 --- a/superset/config.py +++ b/superset/config.py @@ -194,6 +194,13 @@ TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'} ENABLE_CORS = False CORS_OPTIONS = {} +# Chrome allows up to 6 open connections per domain at a time. When there are more +# than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for +# next available socket. PR #5039 is trying to allow domain sharding for Superset, +# and this feature will be enabled by configuration only (by default Superset +# doesn't allow cross-domain request). +SUPERSET_WEBSERVER_DOMAINS = None + # Allowed format types for upload on Database view # TODO: Add processing of other spreadsheet formats (xls, xlsx etc) ALLOWED_EXTENSIONS = set(['csv']) diff --git a/superset/views/base.py b/superset/views/base.py index 31ab1cefc3..7057a9858c 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -26,6 +26,7 @@ FRONTEND_CONF_KEYS = ( 'ENABLE_JAVASCRIPT_CONTROLS', 'DEFAULT_SQLLAB_LIMIT', 'SQL_MAX_ROW', + 'SUPERSET_WEBSERVER_DOMAINS', )