# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from unittest import mock from superset.db_engine_specs.hive import HiveEngineSpec from tests.db_engine_specs.base_tests import DbEngineSpecTestCase class HiveTests(DbEngineSpecTestCase): def test_0_progress(self): log = """ 17/02/07 18:26:27 INFO log.PerfLogger: 17/02/07 18:26:27 INFO log.PerfLogger: """.split( "\n" ) self.assertEqual(0, HiveEngineSpec.progress(log)) def test_number_of_jobs_progress(self): log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 """.split( "\n" ) self.assertEqual(0, HiveEngineSpec.progress(log)) def test_job_1_launched_progress(self): log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 """.split( "\n" ) self.assertEqual(0, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_1(self): log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% """.split( "\n" ) self.assertEqual(0, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_1_map_40_progress( self ): # pylint: disable=invalid-name log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% """.split( "\n" ) self.assertEqual(10, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_1_map_80_reduce_40_progress( self ): # pylint: disable=invalid-name log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 80%, reduce = 40% """.split( "\n" ) self.assertEqual(30, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_2_stages_progress( self ): # pylint: disable=invalid-name log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 80%, reduce = 40% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-2 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 100%, reduce = 0% """.split( "\n" ) self.assertEqual(12, HiveEngineSpec.progress(log)) def test_job_2_launched_stage_2_stages_progress( self ): # pylint: disable=invalid-name log = """ 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 100%, reduce = 0% 17/02/07 19:15:55 INFO ql.Driver: Launching Job 2 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% """.split( "\n" ) self.assertEqual(60, HiveEngineSpec.progress(log)) def test_hive_error_msg(self): msg = ( '{...} errorMessage="Error while compiling statement: FAILED: ' "SemanticException [Error 10001]: Line 4" ":5 Table not found 'fact_ridesfdslakj'\", statusCode=3, " "sqlState='42S02', errorCode=10001)){...}" ) self.assertEqual( ( "hive error: Error while compiling statement: FAILED: " "SemanticException [Error 10001]: Line 4:5 " "Table not found 'fact_ridesfdslakj'" ), HiveEngineSpec.extract_error_message(Exception(msg)), ) e = Exception("Some string that doesn't match the regex") self.assertEqual(f"hive error: {e}", HiveEngineSpec.extract_error_message(e)) msg = ( "errorCode=10001, " 'errorMessage="Error while compiling statement"), operationHandle' '=None)"' ) self.assertEqual( ("hive error: Error while compiling statement"), HiveEngineSpec.extract_error_message(Exception(msg)), ) def test_hive_get_view_names_return_empty_list( self ): # pylint: disable=invalid-name self.assertEqual( [], HiveEngineSpec.get_view_names(mock.ANY, mock.ANY, mock.ANY) ) def test_convert_dttm(self): dttm = self.get_dttm() self.assertEqual( HiveEngineSpec.convert_dttm("DATE", dttm), "CAST('2019-01-02' AS DATE)" ) self.assertEqual( HiveEngineSpec.convert_dttm("TIMESTAMP", dttm), "CAST('2019-01-02 03:04:05.678900' AS TIMESTAMP)", )