[SIP-14] remove dependency on yarn in favor of npm (#6541)

* [SIP-14] remove dependency on yarn in favor of npm Find more details here https://github.com/apache/incubator-superset/issues/6217 * Add core-js to dev deps * lint * fix cypress * break down cypress_build.sh * fix cypress * Use 'npm ci' in place of 'npm install' * Bump npm req to >=6.5.0 * Enforcing npm@'>=6.5.0' * Using latest lts of nvm * Make cache settings global * remove uneeded 'npm run test' as 'npm run cover' takes care of that * Prefix with 'time' * Trying to upgrade babel-eslint instead of downgrading * upgrading babel-polyfill to '@babel/polyfill' * Moving to babel 7 * remove unused packages
2018-12-21 10:24:49 -08:00 · 2018-12-21 10:24:49 -08:00 · 60ccf3eb9d
parent 8c6ddbc842
commit 60ccf3eb9d
20 changed files with 22695 additions and 12548 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -5,42 +5,24 @@ jobs:
    - language: python
      python: 3.6
      env: TOXENV=cypress-dashboard
-      cache:
-        pip: true
-        yarn: true
-        directories:
-          - ~/.npm
-          - ~/.cache
      services:
        - redis-server
      before_install:
-        - nvm install 8.9
+        - nvm install 10.14.2
    - language: python
      python: 3.6
      env: TOXENV=cypress-explore
-      cache:
-        pip: true
-        yarn: true
-        directories:
-          - ~/.npm
-          - ~/.cache
      services:
        - redis-server
      before_install:
-        - nvm install 8.9
+        - nvm install 10.14.2
    - language: python
      python: 3.6
      env: TOXENV=cypress-sqllab
-      cache:
-        pip: true
-        yarn: true
-        directories:
-          - ~/.npm
-          - ~/.cache
      services:
        - redis-server
      before_install:
-        - nvm install 8.9
+        - nvm install 10.14.2
    - language: python
      python: 3.6
      env: TOXENV=py36-mysql
@ -51,17 +33,10 @@ jobs:
        - mysql -u root -e "DROP DATABASE IF EXISTS superset; CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci"
        - mysql -u root -e "CREATE USER 'mysqluser'@'localhost' IDENTIFIED BY 'mysqluserpassword';"
        - mysql -u root -e "GRANT ALL ON superset.* TO 'mysqluser'@'localhost';"
-    - language: node_js
-      node_js: 8.9
-      cache:
-        yarn: true
+    - language: python
+      env: TOXENV=javascript
      before_install:
-        - cd superset/assets
-      install:
-        - yarn install --frozen-lockfile
-      script:
-        - npm run lint
-        - npm run cover
+        - nvm install 10.14.2
    - language: python
      python: 3.6
      env: TOXENV=flake8
@ -89,3 +64,8 @@ script:
  - tox
 after_success:
  - codecov
+cache:
+  pip: true
+  directories:
+    - ~/.npm
+    - ~/.cache
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -254,12 +254,6 @@ First, be sure you are using recent versions of NodeJS and npm. Using [nvm](http

 #### Prerequisite

-If needed, install yarn
-
-```bash
-npm install -g yarn
-```
-
 #### Installing Dependencies

 Install third-party dependencies listed in `package.json`:
@ -268,8 +262,8 @@ Install third-party dependencies listed in `package.json`:
 # From the root of the repository
 cd superset/assets

-# Install dependencies
-yarn install
+# Install dependencies from `package-lock.json`
+npm ci
 ```

 #### Building
@ -302,7 +296,9 @@ npm run sync-backend

 #### Updating NPM packages

-After adding or upgrading an NPM package by changing `package.json`, you must run `yarn install`, which will regenerate the `yarn.lock` file. Then, be sure to commit the new `yarn.lock` so that other users' builds are reproducible. See [the Yarn docs](https://yarnpkg.com/blog/2016/11/24/lockfiles-for-all/) for more information.
+Use npm in the prescribed way, making sure that
+`superset/assets/package-lock.json` is updated according to `npm`-prescribed
+best practices.

 #### Feature flags

@ -329,7 +325,7 @@ tox -e flake8

 # for javascript
 cd superset/assets
-yarn install
+npm ci
 npm run lint
 ```

@ -373,7 +369,6 @@ We use [Jest](https://jestjs.io/) and [Enzyme](http://airbnb.io/enzyme/) to test

 ```bash
 cd superset/assets
-yarn install
 npm run test
 ```

--- a/UPDATING.md
+++ b/UPDATING.md
@ -3,6 +3,12 @@
 This file documents any backwards-incompatible changes in Superset and
 assists people when migrating to a new version.

+## Superset 0.31.0
+From 0.31.0 onwards, we recommend not using the npm package `yarn` in
+favor of good old `npm install`. While yarn should still work just fine,
+you should probably align to guarantee builds similar to the ones we
+use in testing and across the community in general.
+
 ## Superset 0.29.0
 * India was removed from the "Country Map" visualization as the geojson
  file included in the package was very large
--- a/contrib/docker/Dockerfile
+++ b/contrib/docker/Dockerfile
@ -25,12 +25,6 @@ RUN apt-get install -y vim less postgresql-client redis-tools
 RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
    && apt-get install -y nodejs

-# https://yarnpkg.com/lang/en/docs/install/#debian-stable
-RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
-    && apt-get update \
-    && apt-get install -y yarn
-
 WORKDIR /home/superset

 COPY requirements.txt .
@ -48,11 +42,10 @@ ENV PATH=/home/superset/superset/bin:$PATH \
    PYTHONPATH=/home/superset/superset/:$PYTHONPATH

 RUN cd superset/assets \
-    && yarn --non-interactive --frozen-lockfile --link-duplicates \
-    && yarn run sync-backend \
-    && yarn run build \
-    && rm -rf node_modules \
-    && yarn cache clean
+    && npm ci
+    && npm run sync-backend \
+    && npm run build \
+    && rm -rf node_modules

 COPY contrib/docker/docker-init.sh .
 COPY contrib/docker/docker-entrypoint.sh /entrypoint.sh
--- a/contrib/docker/docker-entrypoint.sh
+++ b/contrib/docker/docker-entrypoint.sh
@ -6,8 +6,8 @@ if [ "$#" -ne 0 ]; then
 elif [ "$SUPERSET_ENV" = "development" ]; then
    superset worker &
    # needed by superset runserver
-    (cd superset/assets/ && yarn && yarn run sync-backend)
-    (cd superset/assets/ && yarn run dev) &
+    (cd superset/assets/ && npm ci && npm run sync-backend)
+    (cd superset/assets/ && npm run dev) &
    flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
 elif [ "$SUPERSET_ENV" = "production" ]; then
    superset worker &
--- a/superset/assets/cypress_build.sh
+++ b/superset/assets/cypress_build.sh
@ -4,10 +4,17 @@ set -e
 cd "$(dirname "$0")"

 #run all the python steps in a background process
-(time /home/travis/build/apache/incubator-superset/superset/bin/superset db upgrade; time /home/travis/build/apache/incubator-superset/superset/bin/superset load_test_users; /home/travis/build/apache/incubator-superset/superset/bin/superset load_examples --load-test-data; time /home/travis/build/apache/incubator-superset/superset/bin/superset init; echo "[completed python build steps]"; flask run -p 8081 --with-threads --reload --debugger) &
+time superset db upgrade
+time superset load_test_users
+time superset load_examples --load-test-data
+time superset init
+echo "[completed python build steps]"
+flask run -p 8081 --with-threads --reload --debugger &

 #block on the longer running javascript process
-(time yarn install --frozen-lockfile; time npm run build; echo "[completed js build steps]")
+time npm ci
+time npm run build
+echo "[completed js build steps]"

 CYPRESS_PATH='cypress/integration/'${1}'/*'
 time npm run cypress run -- --spec "$CYPRESS_PATH" --record false --config video=false
--- a/superset/assets/js_build.sh
+++ b/superset/assets/js_build.sh
@ -3,9 +3,7 @@ set -e
 cd "$(dirname "$0")"
 npm --version
 node --version
-npm install -g yarn
-yarn
-npm run lint
-npm run test
-npm run build
-npm run cover
+time npm ci
+time npm run lint
+time npm run cover  # this also runs the tests, so no need to 'npm run test'
+time npm run build
--- a/superset/assets/package-lock.json
+++ b/superset/assets/package-lock.json
--- a/superset/assets/package.json
+++ b/superset/assets/package.json
@ -62,7 +62,6 @@
    "@vx/responsive": "0.0.172",
    "@vx/scale": "^0.0.165",
    "abortcontroller-polyfill": "^1.1.9",
-    "babel-register": "^6.24.1",
    "bootstrap": "^3.3.6",
    "bootstrap-slider": "^10.0.0",
    "brace": "^0.11.1",
@ -71,13 +70,11 @@
    "d3-array": "^1.2.4",
    "d3-cloud": "^1.2.1",
    "d3-color": "^1.2.0",
-    "d3-format": "^1.3.2",
    "d3-hierarchy": "^1.1.5",
    "d3-sankey": "^0.4.2",
    "d3-scale": "^2.1.2",
    "d3-selection": "^1.3.2",
    "d3-svg-legend": "^1.x",
-    "d3-time-format": "^2.1.3",
    "d3-tip": "^0.9.1",
    "datamaps": "^0.5.8",
    "datatables.net-bs": "^1.10.15",
@ -85,7 +82,6 @@
    "distributions": "^1.0.0",
    "dnd-core": "^2.6.0",
    "dompurify": "^1.0.3",
-    "geojson-extent": "^0.3.2",
    "geolib": "^2.0.24",
    "immutable": "^3.8.2",
    "jquery": "3.1.1",
@ -101,8 +97,6 @@
    "re-resizable": "^4.3.1",
    "react": "^16.4.1",
    "react-ace": "^5.10.0",
-    "react-addons-css-transition-group": "^15.6.0",
-    "react-addons-shallow-compare": "^15.4.2",
    "react-bootstrap": "^0.31.5",
    "react-bootstrap-dialog": "^0.10.0",
    "react-bootstrap-slider": "2.1.5",
@ -117,7 +111,6 @@
    "react-map-gl": "^3.0.4",
    "react-markdown": "^3.3.0",
    "react-redux": "^5.0.2",
-    "react-resizable": "^1.3.3",
    "react-search-input": "^0.11.3",
    "react-select": "1.2.1",
    "react-select-fast-filter-options": "^0.2.1",
@ -141,19 +134,20 @@
    "viewport-mercator-project": "^5.0.0"
  },
  "devDependencies": {
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.2.2",
+    "@babel/polyfill": "^7.0.0",
+    "@babel/register": "^7.0.0",
    "@types/jest": "^23.3.5",
    "@types/react": "^16.4.18",
    "@types/react-dom": "^16.0.9",
-    "babel-cli": "^6.14.0",
-    "babel-core": "^6.10.4",
-    "babel-eslint": "^8.2.2",
+    "babel-eslint": "^10.0.1",
    "babel-jest": "^23.6.0",
    "babel-loader": "^7.1.4",
    "babel-plugin-css-modules-transform": "^1.1.0",
    "babel-plugin-dynamic-import-node": "^1.2.0",
    "babel-plugin-lodash": "^3.3.4",
    "babel-plugin-syntax-dynamic-import": "^6.18.0",
-    "babel-polyfill": "^6.23.0",
    "babel-preset-airbnb": "^2.1.1",
    "babel-preset-env": "^1.7.0",
    "cache-loader": "^1.2.2",
@ -162,7 +156,7 @@
    "cypress": "^3.0.3",
    "enzyme": "^3.3.0",
    "enzyme-adapter-react-16": "^1.1.1",
-    "eslint": "^4.19.0",
+    "eslint": "^4.19.1",
    "eslint-config-airbnb": "^15.0.1",
    "eslint-config-prettier": "^2.9.0",
    "eslint-import-resolver-webpack": "^0.10.1",
@ -181,16 +175,13 @@
    "imports-loader": "^0.7.1",
    "jest": "^23.6.0",
    "jsdom": "9.12.0",
-    "less": "^2.6.1",
+    "less": "^3.9.0",
    "less-loader": "^4.1.0",
    "mini-css-extract-plugin": "^0.4.0",
    "minimist": "^1.2.0",
-    "npm-check-updates": "^2.14.2",
    "optimize-css-assets-webpack-plugin": "^5.0.1",
    "po2json": "^0.4.5",
    "prettier": "^1.12.1",
-    "react-addons-test-utils": "^15.6.2",
-    "react-test-renderer": "^15.6.2",
    "redux-mock-store": "^1.2.3",
    "sinon": "^4.5.0",
    "speed-measure-webpack-plugin": "^1.2.3",
--- a/superset/assets/spec/helpers/shim.js
+++ b/superset/assets/spec/helpers/shim.js
@ -1,5 +1,4 @@
 /* eslint no-native-reassign: 0 */
-import 'babel-polyfill';
 import 'abortcontroller-polyfill/dist/abortcontroller-polyfill-only';
 import jsdom from 'jsdom';
 import { configure } from 'enzyme';
--- a/superset/assets/src/dashboard/components/DashboardBuilder.jsx
+++ b/superset/assets/src/dashboard/components/DashboardBuilder.jsx
@ -184,14 +184,12 @@ class DashboardBuilder extends React.Component {
              )}
            </ParentSize>
          </div>
-
-          {this.props.editMode &&
-            this.props.showBuilderPane && (
-              <BuilderComponentPane
-                topOffset={HEADER_HEIGHT + (topLevelTabs ? TABS_HEIGHT : 0)}
-                toggleBuilderPane={this.props.toggleBuilderPane}
-              />
-            )}
+          {this.props.editMode && this.props.showBuilderPane && (
+            <BuilderComponentPane
+              topOffset={HEADER_HEIGHT + (topLevelTabs ? TABS_HEIGHT : 0)}
+              toggleBuilderPane={this.props.toggleBuilderPane}
+            />
+          )}
        </div>
        <ToastPresenter />
      </StickyContainer>
--- a/superset/assets/src/dashboard/components/DashboardGrid.jsx
+++ b/superset/assets/src/dashboard/components/DashboardGrid.jsx
@ -176,16 +176,15 @@ class DashboardGrid extends React.PureComponent {
                />
              ))}

-          {isResizing &&
-            rowGuideTop && (
-              <div
-                className="grid-row-guide"
-                style={{
-                  top: rowGuideTop,
-                  width,
-                }}
-              />
-            )}
+          {isResizing && rowGuideTop && (
+            <div
+              className="grid-row-guide"
+              style={{
+                top: rowGuideTop,
+                width,
+              }}
+            />
+          )}
        </div>
      </div>
    );
--- a/superset/assets/src/dashboard/components/Header.jsx
+++ b/superset/assets/src/dashboard/components/Header.jsx
@ -248,28 +248,26 @@ class Header extends React.PureComponent {
                </Button>
              )}

-              {editMode &&
-                hasUnsavedChanges && (
-                  <Button
-                    bsSize="small"
-                    bsStyle={popButton ? 'primary' : undefined}
-                    onClick={this.overwriteDashboard}
-                  >
-                    {t('Save changes')}
-                  </Button>
-                )}
+              {editMode && hasUnsavedChanges && (
+                <Button
+                  bsSize="small"
+                  bsStyle={popButton ? 'primary' : undefined}
+                  onClick={this.overwriteDashboard}
+                >
+                  {t('Save changes')}
+                </Button>
+              )}

-              {editMode &&
-                !hasUnsavedChanges && (
-                  <Button
-                    bsSize="small"
-                    onClick={this.toggleEditMode}
-                    bsStyle={undefined}
-                    disabled={!userCanEdit}
-                  >
-                    {t('Switch to view mode')}
-                  </Button>
-                )}
+              {editMode && !hasUnsavedChanges && (
+                <Button
+                  bsSize="small"
+                  onClick={this.toggleEditMode}
+                  bsStyle={undefined}
+                  disabled={!userCanEdit}
+                >
+                  {t('Switch to view mode')}
+                </Button>
+              )}

              {editMode && (
                <UndoRedoKeylisteners
@ -280,17 +278,16 @@ class Header extends React.PureComponent {
            </div>
          )}

-          {!editMode &&
-            !hasUnsavedChanges && (
-              <Button
-                bsSize="small"
-                onClick={this.toggleEditMode}
-                bsStyle={popButton ? 'primary' : undefined}
-                disabled={!userCanEdit}
-              >
-                {t('Edit dashboard')}
-              </Button>
-            )}
+          {!editMode && !hasUnsavedChanges && (
+            <Button
+              bsSize="small"
+              onClick={this.toggleEditMode}
+              bsStyle={popButton ? 'primary' : undefined}
+              disabled={!userCanEdit}
+            >
+              {t('Edit dashboard')}
+            </Button>
+          )}

          <HeaderActionsDropdown
            addSuccessToast={this.props.addSuccessToast}
--- a/superset/assets/src/dashboard/components/HeaderActionsDropdown.jsx
+++ b/superset/assets/src/dashboard/components/HeaderActionsDropdown.jsx
@ -125,17 +125,16 @@ class HeaderActionsDropdown extends React.PureComponent {
          />
        )}

-        {hasUnsavedChanges &&
-          userCanSave && (
-            <div>
-              <MenuItem
-                eventKey="discard"
-                onSelect={HeaderActionsDropdown.discardChanges}
-              >
-                {t('Discard changes')}
-              </MenuItem>
-            </div>
-          )}
+        {hasUnsavedChanges && userCanSave && (
+          <div>
+            <MenuItem
+              eventKey="discard"
+              onSelect={HeaderActionsDropdown.discardChanges}
+            >
+              {t('Discard changes')}
+            </MenuItem>
+          </div>
+        )}

        {userCanSave && <MenuItem divider />}

--- a/superset/assets/src/dashboard/components/SliceAdder.jsx
+++ b/superset/assets/src/dashboard/components/SliceAdder.jsx
@ -223,28 +223,23 @@ class SliceAdder extends React.Component {
            ))}
          </DropdownButton>
        </div>
-
        {this.props.isLoading && <Loading />}
-
-        {!this.props.isLoading &&
-          this.state.filteredSlices.length > 0 && (
-            <List
-              width={376}
-              height={slicesListHeight}
-              rowCount={this.state.filteredSlices.length}
-              deferredMeasurementCache={cache}
-              rowHeight={cache.rowHeight}
-              rowRenderer={this.rowRenderer}
-              searchTerm={this.state.searchTerm}
-              sortBy={this.state.sortBy}
-              selectedSliceIds={this.props.selectedSliceIds}
-            />
-          )}
-
+        {!this.props.isLoading && this.state.filteredSlices.length > 0 && (
+          <List
+            width={376}
+            height={slicesListHeight}
+            rowCount={this.state.filteredSlices.length}
+            deferredMeasurementCache={cache}
+            rowHeight={cache.rowHeight}
+            rowRenderer={this.rowRenderer}
+            searchTerm={this.state.searchTerm}
+            sortBy={this.state.sortBy}
+            selectedSliceIds={this.props.selectedSliceIds}
+          />
+        )}
        {this.props.errorMessage && (
          <div className="error-message">{this.props.errorMessage}</div>
        )}
-
        {/* Drag preview is just a single fixed-position element */}
        <AddSliceDragPreview slices={this.state.filteredSlices} />
      </div>
--- a/superset/assets/src/dashboard/components/gridComponents/Chart.jsx
+++ b/superset/assets/src/dashboard/components/gridComponents/Chart.jsx
@ -191,15 +191,14 @@ class Chart extends React.Component {
          and
             https://github.com/apache/incubator-superset/commit/b6fcc22d5a2cb7a5e92599ed5795a0169385a825
        */}
-        {isExpanded &&
-          slice.description_markeddown && (
-            <div
-              className="slice_description bs-callout bs-callout-default"
-              ref={this.setDescriptionRef}
-              // eslint-disable-next-line react/no-danger
-              dangerouslySetInnerHTML={{ __html: slice.description_markeddown }}
-            />
-          )}
+        {isExpanded && slice.description_markeddown && (
+          <div
+            className="slice_description bs-callout bs-callout-default"
+            ref={this.setDescriptionRef}
+            // eslint-disable-next-line react/no-danger
+            dangerouslySetInnerHTML={{ __html: slice.description_markeddown }}
+          />
+        )}

        <div
          className={cx(
--- a/superset/assets/src/dashboard/components/gridComponents/Tabs.jsx
+++ b/superset/assets/src/dashboard/components/gridComponents/Tabs.jsx
@ -156,15 +156,12 @@ class Tabs extends React.PureComponent {
          dragSourceRef: tabsDragSourceRef,
        }) => (
          <div className="dashboard-component dashboard-component-tabs">
-            {editMode &&
-              renderHoverMenu && (
-                <HoverMenu innerRef={tabsDragSourceRef} position="left">
-                  <DragHandle position="left" />
-                  <DeleteComponentButton
-                    onDelete={this.handleDeleteComponent}
-                  />
-                </HoverMenu>
-              )}
+            {editMode && renderHoverMenu && (
+              <HoverMenu innerRef={tabsDragSourceRef} position="left">
+                <DragHandle position="left" />
+                <DeleteComponentButton onDelete={this.handleDeleteComponent} />
+              </HoverMenu>
+            )}

            <BootstrapTabs
              id={tabsComponent.id}
@ -213,13 +210,12 @@ class Tabs extends React.PureComponent {
                </BootstrapTab>
              ))}

-              {editMode &&
-                tabIds.length < MAX_TAB_COUNT && (
-                  <BootstrapTab
-                    eventKey={NEW_TAB_INDEX}
-                    title={<div className="fa fa-plus" />}
-                  />
-                )}
+              {editMode && tabIds.length < MAX_TAB_COUNT && (
+                <BootstrapTab
+                  eventKey={NEW_TAB_INDEX}
+                  title={<div className="fa fa-plus" />}
+                />
+              )}
            </BootstrapTabs>

            {/* don't indicate that a drop on root is allowed when tabs already exist */}
--- a/superset/assets/src/dashboard/components/menu/WithPopoverMenu.jsx
+++ b/superset/assets/src/dashboard/components/menu/WithPopoverMenu.jsx
@ -102,17 +102,15 @@ class WithPopoverMenu extends React.PureComponent {
        style={style}
      >
        {children}
-        {editMode &&
-          isFocused &&
-          menuItems.length > 0 && (
-            <div className="popover-menu">
-              {menuItems.map((node, i) => (
-                <div className="menu-item" key={`menu-item-${i}`}>
-                  {node}
-                </div>
-              ))}
-            </div>
-          )}
+        {editMode && isFocused && menuItems.length > 0 && (
+          <div className="popover-menu">
+            {menuItems.map((node, i) => (
+              <div className="menu-item" key={`menu-item-${i}`}>
+                {node}
+              </div>
+            ))}
+          </div>
+        )}
      </div>
    );
  }
--- a/superset/assets/yarn.lock
+++ b/superset/assets/yarn.lock
--- a/tox.ini
+++ b/tox.ini
@ -47,6 +47,8 @@ whitelist_externals =

 [testenv:cypress-dashboard]
 commands =
+    npm install -g npm@'>=6.5.0'
+    pip install -e {toxinidir}/
    {toxinidir}/superset/assets/cypress_build.sh dashboard
 setenv =
    PYTHONPATH = {toxinidir}
@ -58,6 +60,8 @@ deps =

 [testenv:cypress-explore]
 commands =
+    npm install -g npm@'>=6.5.0'
+    pip install -e {toxinidir}/
    {toxinidir}/superset/assets/cypress_build.sh explore
 setenv =
    PYTHONPATH = {toxinidir}
@ -69,6 +73,8 @@ deps =

 [testenv:cypress-sqllab]
 commands =
+    npm install -g npm@'>=6.5.0'
+    pip install -e {toxinidir}/
    {toxinidir}/superset/assets/cypress_build.sh sqllab
 setenv =
    PYTHONPATH = {toxinidir}
@ -92,7 +98,7 @@ deps =

 [testenv:javascript]
 commands =
-    npm install -g npm@'>=5.6.0'
+    npm install -g npm@'>=6.5.0'
    {toxinidir}/superset/assets/js_build.sh
 deps =