Merge f33cd8a6ac into 5e98439f6d

utf8proc contains all the data which is currently in
unicode_tables.generated.h internally, but in quite a different format.
Ideally unicode_tables.generated.h should be removed as well so we rely
solely on utf8proc. We want to avoid a situation where the possibility
of unicode mismatch occurs, e.g a distro using both unicode 12 and
unicode 13.

cmake.deps/CMakeLists.txt

@@ -37,6 +37,7 @@ option(USE_BUNDLED_MSGPACK "Use the bundled msgpack." ${USE_BUNDLED})
 option(USE_BUNDLED_TS "Use the bundled treesitter runtime." ${USE_BUNDLED})
 option(USE_BUNDLED_TS_PARSERS "Use the bundled treesitter parsers." ${USE_BUNDLED})
 option(USE_BUNDLED_UNIBILIUM "Use the bundled unibilium." ${USE_BUNDLED})
+option(USE_BUNDLED_UTF8PROC "Use the bundled utf8proc library." ${USE_BUNDLED})
 if(USE_BUNDLED AND MSVC)
   option(USE_BUNDLED_GETTEXT "Use the bundled version of gettext." ON)
   option(USE_BUNDLED_LIBICONV "Use the bundled version of libiconv." ON)
@@ -159,6 +160,10 @@ if(USE_BUNDLED_TS)
   include(BuildTreesitter)
 endif()
 
+if(USE_BUNDLED_UTF8PROC)
+  include(BuildUTF8proc)
+endif()
+
 if(WIN32)
   include(GetBinaryDeps)
 

cmake.deps/CMakePresets.json

@@ -17,7 +17,8 @@
       "cacheVariables": {
         "USE_BUNDLED":"OFF",
         "USE_BUNDLED_LIBVTERM":"ON",
-        "USE_BUNDLED_TS":"ON"
+        "USE_BUNDLED_TS":"ON",
+        "USE_BUNDLED_UTF8PROC":"ON"
       },
       "inherits": ["base"]
     }

cmake.deps/cmake/BuildUTF8proc.cmake

@@ -0,0 +1,5 @@
+get_externalproject_options(utf8proc ${DEPS_IGNORE_SHA})
+ExternalProject_Add(utf8proc
+  DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/utf8proc
+  CMAKE_ARGS ${DEPS_CMAKE_ARGS}
+  ${EXTERNALPROJECT_OPTIONS})

cmake.deps/deps.txt

@@ -41,6 +41,9 @@ GETTEXT_SHA256 66415634c6e8c3fa8b71362879ec7575e27da43da562c798a8a2f223e6e47f5c
 LIBICONV_URL https://github.com/neovim/deps/raw/b9bf36eb31f27e8136d907da38fa23518927737e/opt/libiconv-1.17.tar.gz
 LIBICONV_SHA256 8f74213b56238c85a50a5329f77e06198771e70dd9a739779f4c02f65d971313
 
+UTF8PROC_URL https://github.com/JuliaStrings/utf8proc/archive/v2.9.0.tar.gz
+UTF8PROC_SHA256 18c1626e9fc5a2e192311e36b3010bfc698078f692888940f1fa150547abb0c1
+
 TREESITTER_C_URL https://github.com/tree-sitter/tree-sitter-c/archive/v0.21.0.tar.gz
 TREESITTER_C_SHA256 6f0f5d1b71cf8ffd8a37fb638c6022fa1245bd630150b538547d52128ce0ea7e
 TREESITTER_LUA_URL https://github.com/tree-sitter-grammars/tree-sitter-lua/archive/v0.1.0.tar.gz

cmake/FindUTF8proc.cmake

@@ -0,0 +1,12 @@
+find_path2(UTF8PROC_INCLUDE_DIR utf8proc.h)
+find_library2(UTF8PROC_LIBRARY NAMES utf8proc utf8proc_static)
+find_package_handle_standard_args(UTF8proc DEFAULT_MSG
+  UTF8PROC_LIBRARY UTF8PROC_INCLUDE_DIR)
+mark_as_advanced(UTF8PROC_LIBRARY UTF8PROC_INCLUDE_DIR)
+
+add_library(utf8proc INTERFACE)
+target_include_directories(utf8proc SYSTEM BEFORE INTERFACE ${UTF8PROC_INCLUDE_DIR})
+target_link_libraries(utf8proc INTERFACE ${UTF8PROC_LIBRARY})
+
+#TODO(dundargoc): this is a hack that should ideally be hardcoded into the utf8proc project via configure_command
+target_compile_definitions(utf8proc INTERFACE "UTF8PROC_STATIC")

runtime/lua/vim/_defaults.lua

@@ -183,26 +183,22 @@ do
   do
     vim.keymap.set('n', ']d', function()
       vim.diagnostic.goto_next({ float = false })
-    end, {
-      desc = 'Jump to the next diagnostic with the highest severity',
-    })
+    end, { desc = 'Jump to the next diagnostic' })
 
     vim.keymap.set('n', '[d', function()
       vim.diagnostic.goto_prev({ float = false })
-    end, {
-      desc = 'Jump to the previous diagnostic with the highest severity',
-    })
+    end, { desc = 'Jump to the previous diagnostic' })
 
     vim.keymap.set('n', '<C-W>d', function()
       vim.diagnostic.open_float()
-    end, {
-      desc = 'Open a floating window showing diagnostics under the cursor',
-    })
+    end, { desc = 'Show diagnostics under the cursor' })
 
-    vim.keymap.set('n', '<C-W><C-D>', '<C-W>d', {
-      remap = true,
-      desc = 'Open a floating window showing diagnostics under the cursor',
-    })
+    vim.keymap.set(
+      'n',
+      '<C-W><C-D>',
+      '<C-W>d',
+      { remap = true, desc = 'Show diagnostics under the cursor' }
+    )
   end
 end
 

src/nvim/CMakeLists.txt

@@ -35,6 +35,7 @@ find_package(Lpeg REQUIRED)
 find_package(Msgpack 1.0.0 REQUIRED)
 find_package(Treesitter 0.20.9 REQUIRED)
 find_package(Unibilium 2.0 REQUIRED)
+find_package(UTF8proc REQUIRED)
 
 target_link_libraries(main_lib INTERFACE
   iconv
@@ -42,7 +43,8 @@ target_link_libraries(main_lib INTERFACE
   lpeg
   msgpack
   treesitter
-  unibilium)
+  unibilium
+  utf8proc)
 target_link_libraries(nlua0 PUBLIC lpeg)
 
 if(ENABLE_LIBINTL)

src/nvim/mbyte.c

@@ -32,6 +32,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <utf8proc.h>
 #include <uv.h>
 #include <wctype.h>
 
@@ -1379,7 +1380,7 @@ int mb_tolower(int a)
 
 bool mb_isupper(int a)
 {
-  return mb_tolower(a) != a;
+  return utf8proc_isupper(a);
 }
 
 bool mb_isalpha(int a)