pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: use a new official CH driver: clickhouse-connect (#22039)

This commit is contained in:
EugeneTorap 2022-11-16 21:53:15 +03:00 committed by GitHub
parent 900f7f915e
commit 38a3fbdc33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 55 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/docs/databases/clickhouse.mdx
View File

@ -10,33 +10,33 @@ version: 1
To use ClickHouse with Superset, you will need to add the following Python library:
```
clickhouse-sqlalchemy>=0.2.2
clickhouse-connect>=0.4.1
```
If running Superset using Docker Compose, add the following to your `./docker/requirements-local.txt` file:
```
clickhouse-sqlalchemy>=0.2.2
clickhouse-connect>=0.4.1
```
The recommended connector library for ClickHouse is
[sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse).
[clickhouse-connect](https://github.com/ClickHouse/clickhouse-connect).
The expected connection string is formatted as follows:
```
clickhouse+native://<user>:<password>@<host>:<port>/<database>[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database}
clickhousedb://<user>:<password>@<host>:<port>/<database>[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database}
```
Here's a concrete example of a real connection string:
```
clickhouse+native://demo:demo@github.demo.trial.altinity.cloud/default?secure=true
clickhousedb://demo:demo@github.demo.trial.altinity.cloud/default?secure=true
```
If you're using Clickhouse locally on your computer, you can get away with using a native protocol URL that
If you're using Clickhouse locally on your computer, you can get away with using a http protocol URL that
uses the default user without a password (and doesn't encrypt the connection):
```
clickhouse+native://localhost/default
clickhousedb://localhost/default
```

2
docs/docs/databases/installing-database-drivers.mdx
View File

@ -35,7 +35,7 @@ A list of some of the recommended packages.
| [Ascend.io](/docs/databases/ascend) | `pip install impyla` | `ascend://{username}:{password}@{hostname}:{port}/{database}?auth_mechanism=PLAIN;use_ssl=true` |
| [Azure MS SQL](/docs/databases/sql-server) | `pip install pymssql` | `mssql+pymssql://UserName@presetSQL:TestPassword@presetSQL.database.windows.net:1433/TestSchema` |
| [Big Query](/docs/databases/bigquery) | `pip install pybigquery` | `bigquery://{project_id}` |
| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-sqlalchemy` | `clickhouse+native://{username}:{password}@{hostname}:{port}/{database}` |
| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb://{username}:{password}@{hostname}:{port}/{database}` |
| [CockroachDB](/docs/databases/cockroachdb) | `pip install cockroachdb` | `cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable` |
| [Dremio](/docs/databases/dremio) | `pip install sqlalchemy_dremio` | `dremio://user:pwd@host:31010/` |
| [Elasticsearch](/docs/databases/elasticsearch) | `pip install elasticsearch-dbapi` | `elasticsearch+http://{user}:{password}@{host}:9200/` |

2
setup.py
View File

@ -130,7 +130,7 @@ setup(
"pybigquery>=0.4.10",
"google-cloud-bigquery>=2.4.0",
],
"clickhouse": ["clickhouse-sqlalchemy>=0.2.2, <0.3"],
"clickhouse": ["clickhouse-connect>=0.4.1, <0.5"],
"cockroachdb": ["cockroachdb>=0.3.5, <0.4"],
"cors": ["flask-cors>=2.0.0"],
"crate": ["crate[sqlalchemy]>=0.26.0, <0.27"],

47
tests/integration_tests/db_engine_specs/clickhouse_tests.py
View File

@ -1,47 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from unittest import mock
import pytest
from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec
from superset.db_engine_specs.exceptions import SupersetDBAPIDatabaseError
from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
class TestClickHouseDbEngineSpec(TestDbEngineSpec):
def test_convert_dttm(self):
dttm = self.get_dttm()
self.assertEqual(
ClickHouseEngineSpec.convert_dttm("DATE", dttm), "toDate('2019-01-02')"
)
self.assertEqual(
ClickHouseEngineSpec.convert_dttm("DATETIME", dttm),
"toDateTime('2019-01-02 03:04:05')",
)
def test_execute_connection_error(self):
from urllib3.exceptions import NewConnectionError
cursor = mock.Mock()
cursor.execute.side_effect = NewConnectionError(
"Dummypool", message="Exception with sensitive data"
)
with pytest.raises(SupersetDBAPIDatabaseError) as ex:
ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1")

46
tests/unit_tests/db_engine_specs/test_clickhouse.py Normal file
View File

@ -0,0 +1,46 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from unittest import mock
import pytest
from tests.unit_tests.fixtures.common import dttm
def test_convert_dttm(dttm: datetime) -> None:
from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec
assert ClickHouseEngineSpec.convert_dttm("DATE", dttm) == "toDate('2019-01-02')"
assert (
ClickHouseEngineSpec.convert_dttm("DATETIME", dttm)
== "toDateTime('2019-01-02 03:04:05')"
)
def test_execute_connection_error() -> None:
from urllib3.exceptions import NewConnectionError
from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec
from superset.db_engine_specs.exceptions import SupersetDBAPIDatabaseError
cursor = mock.Mock()
cursor.execute.side_effect = NewConnectionError(
"Dummypool", "Exception with sensitive data"
)
with pytest.raises(SupersetDBAPIDatabaseError) as ex:
ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1")