allow domain sharding in frontend (#5039)

docs/installation.rst

@@ -559,6 +559,18 @@ The following keys in `superset_config.py` can be specified to configure CORS:
 * ``CORS_OPTIONS``: options passed to Flask-CORS (`documentation <http://flask-cors.corydolphin.com/en/latest/api.html#extension>`)
 
 
+DOMAIN SHARDING
+---------------
+
+Chrome allows up to 6 open connections per domain at a time. When there are more
+than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for
+next available socket. PR (`#5039 <https://github.com/apache/incubator-superset/pull/5039>`) adds domain sharding to Superset,
+and this feature will be enabled by configuration only (by default Superset
+doesn't allow cross-domain request).
+
+*``SUPERSET_WEBSERVER_DOMAINS``: list of allowed hostnames for domain sharding feature. default `None`
+
+
 MIDDLEWARE
 ----------
 

superset/assets/spec/javascripts/explore/utils_spec.jsx

@@ -1,5 +1,8 @@
+import sinon from 'sinon';
+
 import URI from 'urijs';
 import { getExploreUrlAndPayload, getExploreLongUrl } from '../../../src/explore/exploreUtils';
+import * as hostNamesConfig from '../../../src/utils/hostNamesConfig';
 
 describe('exploreUtils', () => {
   const location = window.location;
@@ -128,6 +131,71 @@ describe('exploreUtils', () => {
     });
   });
 
+  describe('domain sharding', () => {
+    let stub;
+    const availableDomains = [
+      'http://localhost/',
+      'domain1.com', 'domain2.com', 'domain3.com',
+    ];
+    beforeEach(() => {
+      stub = sinon.stub(hostNamesConfig, 'availableDomains').value(availableDomains);
+    });
+    afterEach(() => {
+      stub.restore();
+    });
+
+    it('generate url to different domains', () => {
+      let url = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'json',
+        allowDomainSharding: true,
+      }).url;
+      expect(url).toMatch(availableDomains[0]);
+
+      url = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'json',
+        allowDomainSharding: true,
+      }).url;
+      expect(url).toMatch(availableDomains[1]);
+
+      url = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'json',
+        allowDomainSharding: true,
+      }).url;
+      expect(url).toMatch(availableDomains[2]);
+
+      url = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'json',
+        allowDomainSharding: true,
+      }).url;
+      expect(url).toMatch(availableDomains[3]);
+
+      // circle back to first available domain
+      url = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'json',
+        allowDomainSharding: true,
+      }).url;
+      expect(url).toMatch(availableDomains[0]);
+    });
+    it('not generate url to different domains without flag', () => {
+      let csvURL = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'csv',
+      }).url;
+      expect(csvURL).toMatch(availableDomains[0]);
+
+      csvURL = getExploreUrlAndPayload({
+        formData,
+        endpointType: 'csv',
+      }).url;
+      expect(csvURL).toMatch(availableDomains[0]);
+    });
+  });
+
   describe('getExploreLongUrl', () => {
     it('generates proper base url with form_data', () => {
       compareURI(

superset/assets/src/chart/chartAction.js

@@ -8,6 +8,7 @@ import { requiresQuery, ANNOTATION_SOURCE_TYPES } from '../modules/AnnotationTyp
 import { addDangerToast } from '../messageToasts/actions';
 import { Logger, LOG_ACTIONS_LOAD_CHART } from '../logger';
 import getClientErrorObject from '../utils/getClientErrorObject';
+import { allowCrossDomain } from '../utils/hostNamesConfig';
 
 export const CHART_UPDATE_STARTED = 'CHART_UPDATE_STARTED';
 export function chartUpdateStarted(queryController, latestQueryFormData, key) {
@@ -145,6 +146,7 @@ export function runQuery(formData, force = false, timeout = 60, key) {
       formData,
       endpointType: 'json',
       force,
+      allowDomainSharding: true,
     });
     const logStart = Logger.getTimestamp();
     const controller = new AbortController();
@@ -152,12 +154,20 @@ export function runQuery(formData, force = false, timeout = 60, key) {
 
     dispatch(chartUpdateStarted(controller, payload, key));
 
-    const queryPromise = SupersetClient.post({
+    let querySettings = {
       url,
       postPayload: { form_data: payload },
       signal,
       timeout: timeout * 1000,
-    })
+    };
+    if (allowCrossDomain) {
+      querySettings = {
+        ...querySettings,
+        mode: 'cors',
+        credentials: 'include',
+      };
+    }
+    const queryPromise = SupersetClient.post(querySettings)
       .then(({ json }) => {
         Logger.append(LOG_ACTIONS_LOAD_CHART, {
           slice_id: key,

superset/assets/src/explore/exploreUtils.js

@@ -1,11 +1,23 @@
 /* eslint camelcase: 0 */
 import URI from 'urijs';
+import { availableDomains } from '../utils/hostNamesConfig';
 
 export function getChartKey(explore) {
   const slice = explore.slice;
   return slice ? (slice.slice_id) : 0;
 }
 
+let requestCounter = 0;
+function getHostName(allowDomainSharding = false) {
+  let currentIndex = 0;
+  if (allowDomainSharding) {
+    currentIndex = requestCounter % availableDomains.length;
+    requestCounter += 1;
+  }
+
+  return availableDomains[currentIndex];
+}
+
 export function getAnnotationJsonUrl(slice_id, form_data, isNative) {
   if (slice_id === null || slice_id === undefined) {
     return null;
@@ -49,6 +61,7 @@ export function getExploreUrlAndPayload({
   force = false,
   curUrl = null,
   requestParams = {},
+  allowDomainSharding = false,
 }) {
   if (!formData.datasource) {
     return null;
@@ -57,7 +70,13 @@ export function getExploreUrlAndPayload({
   // The search params from the window.location are carried through,
   // but can be specified with curUrl (used for unit tests to spoof
   // the window.location).
-  let uri = new URI([location.protocol, '//', location.host].join(''));
+  let uri = new URI({
+    protocol: location.protocol.slice(0, -1),
+    hostname: getHostName(allowDomainSharding),
+    port: location.port ? location.port : '',
+    path: '/',
+  });
+
   if (curUrl) {
     uri = URI(URI(curUrl).search());
   }
@@ -105,7 +124,11 @@ export function getExploreUrlAndPayload({
 }
 
 export function exportChart(formData, endpointType) {
-  const { url, payload } = getExploreUrlAndPayload({ formData, endpointType });
+  const { url, payload } = getExploreUrlAndPayload({
+    formData,
+    endpointType,
+    allowDomainSharding: false,
+  });
 
   const exploreForm = document.createElement('form');
   exploreForm.action = url;

superset/assets/src/utils/hostNamesConfig.js

@@ -0,0 +1,23 @@
+function getDomainsConfig() {
+  const appContainer = document.getElementById('app');
+  if (!appContainer) {
+    return [];
+  }
+
+  const bootstrapData = JSON.parse(appContainer.getAttribute('data-bootstrap'));
+  const availableDomains = new Set([location.hostname]);
+  if (bootstrapData &&
+    bootstrapData.common &&
+    bootstrapData.common.conf &&
+    bootstrapData.common.conf.SUPERSET_WEBSERVER_DOMAINS
+  ) {
+    bootstrapData.common.conf.SUPERSET_WEBSERVER_DOMAINS.forEach((hostName) => {
+      availableDomains.add(hostName);
+    });
+  }
+  return Array.from(availableDomains);
+}
+
+export const availableDomains = getDomainsConfig();
+
+export const allowCrossDomain = availableDomains.length > 1;

superset/config.py

@@ -194,6 +194,13 @@ TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'}
 ENABLE_CORS = False
 CORS_OPTIONS = {}
 
+# Chrome allows up to 6 open connections per domain at a time. When there are more
+# than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for
+# next available socket. PR #5039 is trying to allow domain sharding for Superset,
+# and this feature will be enabled by configuration only (by default Superset
+# doesn't allow cross-domain request).
+SUPERSET_WEBSERVER_DOMAINS = None
+
 # Allowed format types for upload on Database view
 # TODO: Add processing of other spreadsheet formats (xls, xlsx etc)
 ALLOWED_EXTENSIONS = set(['csv'])

superset/views/base.py

@@ -26,6 +26,7 @@ FRONTEND_CONF_KEYS = (
     'ENABLE_JAVASCRIPT_CONTROLS',
     'DEFAULT_SQLLAB_LIMIT',
     'SQL_MAX_ROW',
+    'SUPERSET_WEBSERVER_DOMAINS',
 )