refactor(monorepo): move superset-ui to superset(stage 2) (#17552)

.codecov.yml

@@ -3,6 +3,7 @@ codecov:
         after_n_builds: 4
 ignore:
   - "superset/migrations/versions/*.py"
+  - "superset-frontend/packages/superset-ui-demo/**/*"
   - "**/*.stories.tsx"
   - "**/*.stories.jsx"
 coverage:
@@ -14,6 +15,17 @@ coverage:
         # project coverage decrease:
         target: auto
         threshold: 0%
+      core-packages-ts:
+        target: 95%
+        paths:
+          - 'superset-frontend/packages'
+          - '!superset-frontend/packages/**/*.jsx'
+          - '!superset-frontend/packages/**/*.tsx'
+      core-packages-tsx:
+        target: 50%
+        paths:
+          - 'superset-frontend/packages/**/*.jsx'
+          - 'superset-frontend/packages/**/*.tsx'
     patch:
       default:
         informational: true

.github/workflows/superset-frontend.yml

@@ -43,11 +43,24 @@ jobs:
         run: |
           npm run lint
           npm run prettier-check
+      - name: Build plugins packages
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: npm run plugins:build
+      - name: Build plugins Storybook
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: npm run plugins:build-storybook
       - name: unit tests
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: |
           npm run test -- --coverage
+      # todo: remove this step when fix generator as a project in root jest.config.js
+      - name: generator-superset unit tests
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend/packages/generator-superset
+        run: npx jest
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend

.gitignore

@@ -60,14 +60,29 @@ tmp
 rat-results.txt
 superset/app/
 
-# Node.js, webpack artifacts
+# Node.js, webpack artifacts, storybook
 *.entry.js
 *.js.map
 node_modules
 npm-debug.log*
 superset/static/assets
 superset/static/version_info.json
+superset-frontend/**/esm/*
+superset-frontend/**/lib/*
+superset-frontend/**/storybook-static/*
 yarn-error.log
+*.map
+*.min.js
+test-changelog.md
+*.tsbuildinfo
+
+# Ignore package-lock in packages
+plugins/*/package-lock.json
+packages/*/package-lock.json
+
+# For country map geojson conversion script
+.ipynb_checkpoints/
+scripts/*.zip
 
 # IntelliJ
 *.iml

Dockerfile

@@ -57,13 +57,11 @@ ENV BUILD_CMD=${NPM_BUILD_CMD}
 RUN mkdir -p /app/superset-frontend
 RUN mkdir -p /app/superset/assets
 COPY ./docker/frontend-mem-nag.sh /
-COPY ./superset-frontend/package* /app/superset-frontend/
+COPY ./superset-frontend /app/superset-frontend
 RUN /frontend-mem-nag.sh \
         && cd /app/superset-frontend \
         && npm ci
 
-# Next, copy in the rest and let webpack do its thing
-COPY ./superset-frontend /app/superset-frontend
 # This seems to be the most expensive step
 RUN cd /app/superset-frontend \
         && npm run ${BUILD_CMD} \

scripts/babel_extract.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+LICENSE_TMP=$(mktemp)
+cat <<'EOF'> "$LICENSE_TMP"
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+EOF
+
+cd $ROOT_DIR
+pybabel extract \
+  -F superset/translations/babel.cfg \
+  -o superset/translations/messages.pot \
+  --sort-output \
+  --copyright-holder=Superset \
+  --project=Superset \
+  -k _ -k __ -k t -k tn -k tct .
+cat $LICENSE_TMP $ROOT_DIR/superset/translations/messages.pot > messages.pot.tmp \
+  && mv messages.pot.tmp $ROOT_DIR/superset/translations/messages.pot \
+  && cd $CURRENT_DIR

superset-frontend/.eslintignore

@@ -28,4 +28,9 @@ src/dashboard/deprecated/*
 src/temp/*
 **/node_modules
 *.d.ts
+coverage/
+esm/
+lib/
+tmp/
+storybook-static/
 temporary_superset_ui/**/*

superset-frontend/.eslintrc.js

@@ -25,6 +25,18 @@ Object.entries(packageConfig.dependencies).forEach(([pkg]) => {
     importCoreModules.push(pkg);
   }
 });
+
+// ignore files when running ForkTsCheckerWebpackPlugin
+let ignorePatterns = [];
+if (process.env.NODE_ENV === 'production') {
+  ignorePatterns = [
+    '*.test.{js,ts,jsx,tsx}',
+    'plugins/**/test/**/*',
+    'packages/**/test/**/*',
+    'packages/generator-superset/**/*',
+  ];
+}
+
 module.exports = {
   extends: [
     'airbnb',
@@ -307,4 +319,5 @@ module.exports = {
     'react/static-property-placement': 0, // disabled temporarily
     'prettier/prettier': 'error',
   },
+  ignorePatterns,
 };

superset-frontend/babel.config.js

@@ -38,6 +38,14 @@ module.exports = {
       '@babel/preset-react',
       { development: process.env.BABEL_ENV === 'development' },
     ],
+    '@babel/preset-typescript',
+    [
+      '@emotion/babel-preset-css-prop',
+      {
+        autoLabel: 'dev-only',
+        labelFormat: '[local]',
+      },
+    ],
   ],
   plugins: [
     'lodash',
@@ -47,6 +55,8 @@ module.exports = {
     ['@babel/plugin-proposal-private-methods', { loose: true }],
     ['@babel/plugin-proposal-nullish-coalescing-operator', { loose: true }],
     ['@babel/plugin-transform-runtime', { corejs: 3 }],
+    // only used in packages/superset-ui-core/src/chart/components/reactify.tsx
+    ['babel-plugin-typescript-to-proptypes', { loose: true }],
     'react-hot-loader/babel',
   ],
   env: {
@@ -70,7 +80,14 @@ module.exports = {
     },
     // build instrumented code for testing code coverage with Cypress
     instrumented: {
-      plugins: ['istanbul'],
+      plugins: [
+        [
+          'istanbul',
+          {
+            exclude: ['plugins/**/*', 'packages/**/*'],
+          },
+        ],
+      ],
     },
     production: {
       plugins: [

superset-frontend/jest.config.js

@@ -16,23 +16,48 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 module.exports = {
-  testRegex: '(\\/spec|\\/src)\\/.*(_spec|\\.test)\\.(j|t)sx?$',
+  testRegex: '\\/(spec|src|plugins|packages)\\/.*(_spec|\\.test)\\.[jt]sx?$',
+  testPathIgnorePatterns: [
+    'packages\\/generator-superset\\/.*',
+    '/node_modules/',
+  ],
   moduleNameMapper: {
     '\\.(css|less|geojson)$': '<rootDir>/spec/__mocks__/mockExportObject.js',
     '\\.(gif|ttf|eot|png|jpg)$': '<rootDir>/spec/__mocks__/mockExportString.js',
     '\\.svg$': '<rootDir>/spec/__mocks__/svgrMock.tsx',
     '^src/(.*)$': '<rootDir>/src/$1',
     '^spec/(.*)$': '<rootDir>/spec/$1',
+    // mapping to souce code instead of lib or esm module
+    '@superset-ui/(((?!(legacy-preset-chart-deckgl|core/src)).)*)$':
+      '<rootDir>/node_modules/@superset-ui/$1/src',
+    '@superset-ui/core/src/(.*)$':
+      '<rootDir>/node_modules/@superset-ui/core/src/$1',
   },
   testEnvironment: 'jsdom',
   modulePathIgnorePatterns: ['<rootDir>/temporary_superset_ui'],
   setupFilesAfterEnv: ['<rootDir>/spec/helpers/setup.ts'],
   testURL: 'http://localhost',
-  collectCoverageFrom: ['src/**/*.{js,jsx,ts,tsx}', '!**/*.stories.*'],
+  collectCoverageFrom: [
+    'src/**/*.{js,jsx,ts,tsx}',
+    '{packages,plugins}/**/src/**/*.{js,jsx,ts,tsx}',
+    '!**/*.stories.*',
+    '!packages/superset-ui-demo/**/*',
+  ],
   coverageDirectory: '<rootDir>/coverage/',
+  coveragePathIgnorePatterns: [
+    'coverage/',
+    'node_modules/',
+    'public/',
+    'tmp/',
+    'dist/',
+  ],
+  coverageReporters: ['lcov', 'json-summary', 'html'],
   transform: {
     '^.+\\.jsx?$': 'babel-jest',
+    // ts-jest can't load plugin 'babel-plugin-typescript-to-proptypes'
+    'reactify\\.tsx$': 'babel-jest',
     '^.+\\.tsx?$': 'ts-jest',
   },
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
@@ -40,9 +65,15 @@ module.exports = {
   globals: {
     'ts-jest': {
       babelConfig: true,
+      // todo: duo to packages/**/test and plugins/**/test lack of type checking
+      // turning off checking in Jest.
+      // https://kulshekhar.github.io/ts-jest/docs/getting-started/options/isolatedModules
+      isolatedModules: true,
       diagnostics: {
         warnOnly: true,
       },
     },
+    __DEV__: true,
+    caches: true,
   },
 };

superset-frontend/lerna.json

@@ -1,7 +1,7 @@
 {
   "lerna": "3.2.1",
   "npmClient": "npm",
-  "packages": ["packages/*", "plugins/*", "temporary_superset_ui/*"],
+  "packages": ["packages/*", "plugins/*"],
   "useWorkspaces": true,
   "version": "0.0.0",
   "ignoreChanges": [

superset-frontend/package.json

@@ -28,22 +28,37 @@
     "doc": "docs",
     "test": "spec"
   },
+  "workspaces": [
+    "packages/*",
+    "plugins/*"
+  ],
   "scripts": {
+    "_lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,tsx .",
+    "_prettier": "prettier './({src,spec,cypress-base,plugins,packages}/**/*{.js,.jsx,.ts,.tsx,.css,.less,.scss,.sass}|package.json)'",
     "build": "cross-env NODE_OPTIONS=--max_old_space_size=8192 NODE_ENV=production BABEL_ENV=\"${BABEL_ENV:=production}\" webpack --mode=production --color",
     "build-dev": "cross-env NODE_OPTIONS=--max_old_space_size=8192 NODE_ENV=development webpack --mode=development --color",
     "build-instrumented": "cross-env NODE_ENV=production BABEL_ENV=instrumented webpack --mode=production --color",
     "build-storybook": "build-storybook",
     "check-translation": "prettier --check ../superset/translations/**/LC_MESSAGES/*.json",
-    "clean-css": "prettier --write 'src/**/*.{css,less,sass,scss}'",
     "clean-translation": "prettier --write ../superset/translations/**/LC_MESSAGES/*.json",
     "cover": "cross-env NODE_ENV=test jest --coverage",
     "dev": "webpack --mode=development --color --watch",
     "dev-server": "cross-env NODE_ENV=development BABEL_ENV=development node --max_old_space_size=4096 ./node_modules/webpack-dev-server/bin/webpack-dev-server.js --mode=development",
-    "format": "prettier --write './{src,spec,cypress-base,plugins,packages}/**/*{.js,.jsx,.ts,.tsx,.css,.less,.scss,.sass}'",
-    "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && npm run type",
-    "lint-fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,tsx . && npm run clean-css && npm run type",
-    "prettier": "npm run format",
-    "prettier-check": "prettier --check 'src/**/*.{css,less,sass,scss}'",
+    "format": "npm run _prettier -- --write",
+    "lint": "npm run _lint && npm run type",
+    "lint-fix": "npm run _lint -- --fix && npm run type",
+    "plugins:build": "node ./scripts/build.js",
+    "plugins:build-assets": "node ./scripts/copyAssets.js",
+    "plugins:build-storybook": "cd packages/superset-ui-demo && npm run build-storybook",
+    "plugins:chromatic": "cd packages/superset-ui-demo && npm run chromatic",
+    "plugins:create-conventional-version": "npm run prune && ./scripts/lernaVersion.sh '--conventional-commits --create-release github'",
+    "plugins:create-minor-version": "npm run prune && ./scripts/lernaVersion.sh minor",
+    "plugins:create-patch-version": "npm run prune && ./scripts/lernaVersion.sh patch",
+    "plugins:release-conventional": "npm run prune && lerna publish --conventional-commits --create-release github --yes",
+    "plugins:release-from-tag": "npm run prune && lerna publish from-package --yes",
+    "plugins:storybook": "cd packages/superset-ui-demo && npm run storybook",
+    "prettier": "npm run _prettier -- --write",
+    "prettier-check": "npm run _prettier -- --check",
     "prod": "npm run build",
     "prune": "rm -rf ./{packages,plugins}/*/{lib,esm,tsconfig.tsbuildinfo,package-lock.json}",
     "storybook": "cross-env NODE_ENV=development BABEL_ENV=development start-storybook -s ./src/assets/images -p 6006",
@@ -115,6 +130,7 @@
     "abortcontroller-polyfill": "^1.1.9",
     "antd": "^4.9.4",
     "array-move": "^2.2.1",
+    "babel-plugin-typescript-to-proptypes": "^2.0.0",
     "bootstrap": "^3.4.1",
     "bootstrap-slider": "^10.0.0",
     "brace": "^0.11.1",

superset-frontend/packages/generator-superset/jest.config.js

@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+module.exports = {
+  displayName: 'generator-superset',
+  testRegex: 'test\\/.*\\.test\\.[jt]sx?$',
+  testEnvironment: 'node',
+};

superset-frontend/packages/generator-superset/package.json

@@ -23,7 +23,8 @@
   ],
   "devDependencies": {
     "yeoman-assert": "^3.1.0",
-    "yeoman-test": "^2.0.0"
+    "yeoman-test": "^6.2.0",
+    "fs-extra": "^10.0.0"
   },
   "engines": {
     "npm": ">= 4.0.0"

superset-frontend/packages/generator-superset/test/app.test.js

@@ -17,43 +17,22 @@
  * under the License.
  */
 
-const path = require('path');
 const assert = require('yeoman-assert');
 const helpers = require('yeoman-test');
+const appModule = require('../generators/app');
 
-describe('generator-superset:app', () => {
-  let dir;
-
-  beforeAll(() => {
-    dir = process.cwd();
-
-    return helpers.run(path.join(__dirname, '../generators/app')).withPrompts({
+test('generator-superset:app:creates files', () =>
+  helpers
+    .run(appModule)
+    .withPrompts({
       subgenerator: 'package',
       name: 'my-package',
-    });
-  });
-
-  /*
-   * Change working directory back to original working directory
-   * after the test has completed.
-   * yeoman tests switch to tmp directory and write files there.
-   * Usually this is fine for solo package.
-   * However, for a monorepo like this one,
-   * it made jest confuses with current directory
-   * (being in tmp directory instead of superset-ui root)
-   * and interferes with other tests in sibling packages
-   * that are run after the yeoman tests.
-   */
-  afterAll(() => {
-    process.chdir(dir);
-  });
-
-  it('creates files', () => {
-    assert.file([
-      'package.json',
-      'README.md',
-      'src/index.ts',
-      'test/index.test.ts',
-    ]);
-  });
-});
+    })
+    .then(function () {
+      assert.file([
+        'package.json',
+        'README.md',
+        'src/index.ts',
+        'test/index.test.ts',
+      ]);
+    }));

superset-frontend/packages/generator-superset/test/package.test.js

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const path = require('path');
+const assert = require('yeoman-assert');
+const helpers = require('yeoman-test');
+const fs = require('fs-extra');
+const packageModule = require('../generators/package');
+
+test('generator-superset:package:creates typescript files', () =>
+  helpers
+    .run(packageModule)
+    .inTmpDir(function (dir) {
+      // `dir` is the path to the new temporary directory
+      fs.copySync(path.join(__dirname, '../generators/package'), dir);
+    })
+    .withPrompts({ name: 'my-package', language: 'typescript' })
+    .withOptions({ skipInstall: true })
+    .then(function () {
+      assert.file([
+        'package.json',
+        'README.md',
+        'src/index.ts',
+        'test/index.test.ts',
+      ]);
+    }));
+
+test('generator-superset:package:creates javascript files', () =>
+  helpers
+    .run(packageModule)
+    .inTmpDir(function (dir) {
+      // `dir` is the path to the new temporary directory
+      fs.copySync(path.join(__dirname, '../generators/package'), dir);
+    })
+    .withPrompts({ name: 'my-package', language: 'javascript' })
+    .withOptions({ skipInstall: true })
+    .then(function () {
+      assert.file([
+        'package.json',
+        'README.md',
+        'src/index.js',
+        'test/index.test.js',
+      ]);
+    }));

superset-frontend/packages/generator-superset/test/plugin-chart.test.js

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* eslint-env node */
+const path = require('path');
+const assert = require('yeoman-assert');
+const helpers = require('yeoman-test');
+const fs = require('fs-extra');
+const pluginChartModule = require('../generators/plugin-chart');
+
+test('generator-superset:plugin-chart:creates files', () =>
+  helpers
+    .run(pluginChartModule)
+    .inTmpDir(function (dir) {
+      // `dir` is the path to the new temporary directory
+      fs.copySync(path.join(__dirname, '../generators/plugin-chart'), dir);
+    })
+    .withPrompts({
+      packageName: 'cold-map',
+      description: 'Cold Map',
+      componentType: 'function',
+      chartType: 'regular',
+    })
+    .withOptions({ skipInstall: true })
+    .then(function () {
+      assert.file([
+        'package.json',
+        'README.md',
+        'src/plugin/buildQuery.ts',
+        'src/plugin/controlPanel.ts',
+        'src/plugin/index.ts',
+        'src/plugin/transformProps.ts',
+        'src/ColdMap.tsx',
+        'src/index.ts',
+        'test/index.test.ts',
+        'test/plugin/buildQuery.test.ts',
+        'test/plugin/transformProps.test.ts',
+        'types/external.d.ts',
+        'src/images/thumbnail.png',
+      ]);
+    }));

superset-frontend/packages/generator-superset/tsconfig.json

@@ -5,15 +5,9 @@
     "noEmit": true,
     "rootDir": "."
   },
-  "extends": "../../../tsconfig.json",
+  "extends": "../../tsconfig.json",
   "include": [
     "**/*",
-    "../types/**/*",
-    "../../../types/**/*"
+    "../../types/**/*",
   ],
-  "references": [
-    {
-      "path": ".."
-    }
-  ]
 }