fix: downgrade sqlparse and add unit test (#10165)

* Downgrade sqlparse and add unit test

* Explain why sqlparse is pinned

Co-authored-by: bogdan kyryliuk <bogdankyryliuk@dropbox.com>

requirements.txt

@@ -88,7 +88,7 @@ six==1.14.0               # via bleach, cryptography, flask-jwt-extended, flask-
 slackclient==2.6.2        # via apache-superset (setup.py)
 sqlalchemy-utils==0.36.6  # via apache-superset (setup.py), flask-appbuilder
 sqlalchemy==1.3.16        # via alembic, apache-superset (setup.py), flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
-sqlparse==0.3.1           # via apache-superset (setup.py)
+sqlparse==0.3.0           # via apache-superset (setup.py)
 typing-extensions==3.7.4.2  # via aiohttp
 urllib3==1.25.9           # via selenium
 vine==1.3.0               # via amqp, celery

setup.py

@@ -105,7 +105,7 @@ setup(
         "slackclient>=2.6.2",
         "sqlalchemy>=1.3.16, <2.0",
         "sqlalchemy-utils>=0.36.6,<0.37",
-        "sqlparse>=0.3.0, <0.4",
+        "sqlparse==0.3.0",  # PINNED! see https://github.com/andialbrecht/sqlparse/issues/562
         "wtforms-json",
     ],
     extras_require={

tests/sql_parse_tests.py

@@ -16,6 +16,8 @@
 # under the License.
 import unittest
 
+import sqlparse
+
 from superset.sql_parse import ParsedQuery, Table
 
 
@@ -187,7 +189,11 @@ class TestSupersetSqlParse(unittest.TestCase):
     # SHOW TABLES ((FROM | IN) qualifiedName)? (LIKE pattern=STRING)?
     def test_show_tables(self):
         query = "SHOW TABLES FROM s1 like '%order%'"
-        self.assertEqual(set(), self.extract_tables(query))
+        # TODO: figure out what should code do here
+        self.assertEqual({Table("s1")}, self.extract_tables(query))
+        # Expected behavior is below, it is fixed in sqlparse>=3.1
+        # However sqlparse==3.1 breaks some sql formatting.
+        # self.assertEqual(set(), self.extract_tables(query))
 
     # SHOW COLUMNS (FROM | IN) qualifiedName
     def test_show_columns(self):
@@ -560,3 +566,16 @@ class TestSupersetSqlParse(unittest.TestCase):
         SELECT * FROM match
         """
         self.assertEqual({Table("foo")}, self.extract_tables(query))
+
+    def test_sqlparse_formatting(self):
+        # sqlparse 0.3.1 has a bug and removes space between from and from_unixtime while formatting:
+        # SELECT extract(HOUR\n               fromfrom_unixtime(hour_ts)
+        # AT TIME ZONE 'America/Los_Angeles')\nfrom table
+        self.assertEqual(
+            "SELECT extract(HOUR\n               from from_unixtime(hour_ts) "
+            "AT TIME ZONE 'America/Los_Angeles')\nfrom table",
+            sqlparse.format(
+                "SELECT extract(HOUR from from_unixtime(hour_ts) AT TIME ZONE 'America/Los_Angeles') from table",
+                reindent=True,
+            ),
+        )