Merge 4d0d5a09a1 into 435dee74bb

Problem:  When :edit an existing buffer, line('w$') may return a
          wrong result.
Solution: Reset w_valid in curwin_init() (Jaehwang Jung)

`do_ecmd()` reinitializes the current window (`curwin_init()`) whose
`w_valid` field may have `VALID_BOTLINE` set. Resetting `w_botline`
without marking it as invalid makes subsequent `validate_botline()`
calls a no-op, thus resulting in wrong `line('w$')` value.

closes: vim/vim#14642

eb80b8304e

Co-authored-by: Jaehwang Jung <tomtomjhj@gmail.com>

runtime/doc/builtin.txt

@@ -8208,6 +8208,10 @@ synconcealed({lnum}, {col})                                     *synconcealed()*
 			synconcealed(lnum, 5)   [1, 'X', 2]
 			synconcealed(lnum, 6)   [0, '', 0]
 
+		Note: Doesn't consider |matchadd()| highlighting items,
+		since syntax and matching highlighting are two different
+		mechanisms |syntax-vs-match|.
+
 synstack({lnum}, {col})                                             *synstack()*
 		Return a |List|, which is the stack of syntax items at the
 		position {lnum} and {col} in the current window.  {lnum} is

runtime/doc/news.txt

@@ -283,6 +283,7 @@ The following new APIs and features were added.
     control sequence is used.
   • |vim.treesitter.foldexpr()| now recognizes folds captured using a
     quantified query pattern.
+  • |vim.treesitter.get_highlight()| get highlight information from parser.
 
 • |:Man| now supports `:hide` modifier to open page in the current window.
 

runtime/doc/pattern.txt

@@ -1375,6 +1375,19 @@ Finally, these constructs are unique to Perl:
 ==============================================================================
 10. Highlighting matches				*match-highlight*
 
+							*syntax-vs-match*
+		Note that the match highlight mechanism is independent
+		of |syntax-highlighting|, which is (usually) a buffer-local
+		highlighting, while matching is window-local, both methods
+		can be freely mixed.  Match highlighting functions give you
+		a bit more flexibility in when and how to apply, but are
+		typically only used for temporary highlighting, without strict
+		rules.  Both methods can be used to conceal text.
+
+		Thus the matching functions like |matchadd()| won't consider
+		syntax rules and functions like |synconcealed()| and the
+		other way around.
+
 							*:mat* *:match*
 :mat[ch] {group} /{pattern}/
 		Define a pattern to highlight in the current window.  It will

runtime/doc/syntax.txt

@@ -3825,7 +3825,9 @@ Whether or not it is actually concealed depends on the value of the
 'conceallevel' option.  The 'concealcursor' option is used to decide whether
 concealable items in the current line are displayed unconcealed to be able to
 edit the line.
-Another way to conceal text is with |matchadd()|.
+
+Another way to conceal text is with |matchadd()|, but internally this works a
+bit differently |syntax-vs-match|.
 
 concealends						*:syn-concealends*
 
@@ -3833,7 +3835,9 @@ When the "concealends" argument is given, the start and end matches of
 the region, but not the contents of the region, are marked as concealable.
 Whether or not they are actually concealed depends on the setting on the
 'conceallevel' option. The ends of a region can only be concealed separately
-in this way when they have their own highlighting via "matchgroup"
+in this way when they have their own highlighting via "matchgroup".  The
+|synconcealed()| function can be used to retrieve information about conealed
+items.
 
 cchar							*:syn-cchar*
 							*E844*

runtime/doc/treesitter.txt

@@ -733,6 +733,25 @@ get_captures_at_pos({bufnr}, {row}, {col})
     Return: ~
         (`{capture: string, lang: string, metadata: table}[]`)
 
+get_highlight({parser})                       *vim.treesitter.get_highlight()*
+    Get highlight information from parser.
+
+    Parameters: ~
+      • {parser}  (`vim.treesitter.LanguageTree`)
+
+    Return: ~
+        (`table[]`) A list of objects with the following fields:
+        • {range} (`Range6`) The range of the highlight
+        • {hl_group} (`string`) The group of the highlight
+        • {priority} (`integer`) The priority of the highlight
+        • {conceal} (`string?`) The conceal of the highlight, nil for no
+          conceal
+        • {spell} (`boolean?`) Whether the highlight is spell or not
+          • If `nil` then the highlight doesn't have a spell set
+          • If `true` then the highlight is spell
+          • If `false` then the highlight is nospell
+        • {url} (`string?`) The url of the highlight, nil for no url
+
 get_node({opts})                                   *vim.treesitter.get_node()*
     Returns the smallest named node at the given position
 

runtime/doc/usr_41.txt

@@ -948,7 +948,7 @@ Syntax and highlighting:	  *syntax-functions* *highlighting-functions*
 	synIDattr()		get a specific attribute of a syntax ID
 	synIDtrans()		get translated syntax ID
 	synstack()		get list of syntax IDs at a specific position
-	synconcealed()		get info about concealing
+	synconcealed()		get info about (syntax) concealing
 	diff_hlID()		get highlight ID for diff mode at a position
 	matchadd()		define a pattern to highlight (a "match")
 	matchaddpos()		define a list of positions to highlight

runtime/lua/tohtml.lua

@@ -436,23 +436,22 @@ local function styletable_treesitter(state)
       return
     end
     local root = tstree:root()
-    local q = buf_highlighter:get_query(tree:lang())
     --- @type vim.treesitter.Query?
-    local query = q:query()
+    local query = buf_highlighter:get_query(tree:lang()):query()
     if not query then
       return
     end
-    for capture, node, metadata in query:iter_captures(root, buf_highlighter.bufnr, 0, state.buflen) do
-      local srow, scol, erow, ecol = node:range()
+    for info in
+      buf_highlighter._iter_highlight_info(query, root, buf_highlighter.bufnr, 0, state.buflen)
+    do
+      --- @type integer,integer,integer,integer,integer,integer
+      local srow, scol, _, erow, ecol, _ = unpack(info.range)
       --- @diagnostic disable-next-line: invisible
-      local c = q._query.captures[capture]
-      if c ~= nil then
-        local hlid = register_hl(state, '@' .. c .. '.' .. tree:lang())
-        if metadata.conceal and state.opt.conceallevel ~= 0 then
-          styletable_insert_conceal(state, srow + 1, scol + 1, erow + 1, ecol + 1, metadata.conceal)
-        end
-        styletable_insert_range(state, srow + 1, scol + 1, erow + 1, ecol + 1, hlid)
+      local hlid = register_hl(state, info.hl_group)
+      if info.conceal and state.opt.conceallevel ~= 0 then
+        styletable_insert_conceal(state, srow + 1, scol + 1, erow + 1, ecol + 1, info.conceal)
       end
+      styletable_insert_range(state, srow + 1, scol + 1, erow + 1, ecol + 1, hlid)
     end
   end)
 end

runtime/lua/vim/_meta/vimfn.lua

@@ -9752,6 +9752,10 @@ function vim.fn.synIDtrans(synID) end
 ---   synconcealed(lnum, 5)   [1, 'X', 2]
 ---   synconcealed(lnum, 6)   [0, '', 0]
 ---
+--- Note: Doesn't consider |matchadd()| highlighting items,
+--- since syntax and matching highlighting are two different
+--- mechanisms |syntax-vs-match|.
+---
 --- @param lnum integer
 --- @param col integer
 --- @return {[1]: integer, [2]: string, [3]: integer}

runtime/lua/vim/treesitter.lua

@@ -466,4 +466,27 @@ function M.foldexpr(lnum)
   return M._fold.foldexpr(lnum)
 end
 
+--- Get highlight information from parser.
+---
+---@param parser vim.treesitter.LanguageTree
+---@return vim.treesitter.highlighter.Iter.ret[]
+function M.get_highlight(parser)
+  local highlights = {}
+  parser:for_each_tree(function(tree, ltree)
+    local source = ltree:source()
+    local hl_query = M.query.get(ltree:lang(), 'highlights')
+
+    if not hl_query then
+      return
+    end
+
+    local highlighter = require('vim.treesitter.highlighter')
+    for info in highlighter._iter_highlight_info(hl_query, tree:root(), source, 0, -1) do
+      info.range = { M._range.unpack4(info.range) }
+      table.insert(highlights, info)
+    end
+  end)
+  return highlights
+end
+
 return M

runtime/lua/vim/treesitter/highlighter.lua

@@ -4,12 +4,30 @@ local Range = require('vim.treesitter._range')
 
 local ns = api.nvim_create_namespace('treesitter/highlighter')
 
----@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, TSQueryMatch
+--- @class vim.treesitter.highlighter.Iter.ret
+--- @inlinedoc
+--- The range of the highlight
+--- @field range Range6
+--- The group of the highlight
+--- @field hl_group string
+--- The priority of the highlight
+--- @field priority integer
+--- The conceal of the highlight, nil for no conceal
+--- @field conceal string?
+--- Whether the highlight is spell or not
+---   - If `nil` then the highlight doesn't have a spell set
+---   - If `true` then the highlight is spell
+---   - If `false` then the highlight is nospell
+--- @field spell boolean?
+--- The url of the highlight, nil for no url
+--- @field url string?
+
+---@alias vim.treesitter.highlighter.Iter fun(end_line:integer):vim.treesitter.highlighter.Iter.ret?,TSNode?
 
 ---@class (private) vim.treesitter.highlighter.Query
 ---@field private _query vim.treesitter.Query?
 ---@field private lang string
----@field private hl_cache table<integer,integer>
+---@field private hl_cache table<string,integer>
 local TSHighlighterQuery = {}
 TSHighlighterQuery.__index = TSHighlighterQuery
 
@@ -32,19 +50,18 @@ function TSHighlighterQuery.new(lang, query_string)
 end
 
 ---@package
----@param capture integer
+---@param name string
 ---@return integer?
-function TSHighlighterQuery:get_hl_from_capture(capture)
-  if not self.hl_cache[capture] then
-    local name = self._query.captures[capture]
+function TSHighlighterQuery:get_hl_from_capture(name)
+  if not self.hl_cache[name] then
     local id = 0
     if not vim.startswith(name, '_') then
-      id = api.nvim_get_hl_id_by_name('@' .. name .. '.' .. self.lang)
+      id = api.nvim_get_hl_id_by_name(name)
     end
-    self.hl_cache[capture] = id
+    self.hl_cache[name] = id
   end
 
-  return self.hl_cache[capture]
+  return self.hl_cache[name]
 end
 
 ---@package
@@ -244,11 +261,11 @@ function TSHighlighter:get_query(lang)
 end
 
 --- @param match TSQueryMatch
---- @param bufnr integer
+--- @param source integer|string
 --- @param capture integer
 --- @param metadata vim.treesitter.query.TSMetadata
 --- @return string?
-local function get_url(match, bufnr, capture, metadata)
+local function get_url(match, source, capture, metadata)
   ---@type string|number|nil
   local url = metadata[capture] and metadata[capture].url
 
@@ -266,7 +283,7 @@ local function get_url(match, bufnr, capture, metadata)
   -- from the first.
   local other_node = captures[url][1]
 
-  return vim.treesitter.get_node_text(other_node, bufnr, {
+  return vim.treesitter.get_node_text(other_node, source, {
     metadata = metadata[url],
   })
 end
@@ -283,6 +300,57 @@ local function get_spell(capture_name)
   return nil, 0
 end
 
+---@param hl_query vim.treesitter.Query
+---@param root_node TSNode
+---@param source integer|string
+---@param start_row integer?
+---@param end_row integer?
+---@return vim.treesitter.highlighter.Iter
+function TSHighlighter._iter_highlight_info(hl_query, root_node, source, start_row, end_row)
+  local iter = hl_query:iter_captures(root_node, source, start_row, end_row)
+
+  return function(end_line)
+    local capture, node, metadata, match = iter(end_line)
+    if not node then
+      return
+    end
+
+    if not capture then
+      return nil, node
+    end
+
+    local range = vim.treesitter.get_range(node, source, metadata and metadata[capture])
+
+    local capture_name = hl_query.captures[capture]
+
+    local hl_group = capture_name
+    if not vim.startswith(hl_group, '_') then
+      hl_group = '@' .. hl_group .. '.' .. hl_query.lang
+    end
+
+    local spell, spell_pri_offset = get_spell(capture_name)
+
+    local priority = (
+      tonumber(metadata.priority or metadata[capture] and metadata[capture].priority)
+      or vim.highlight.priorities.treesitter
+    ) + spell_pri_offset
+
+    local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
+
+    local url = get_url(match, source, capture, metadata)
+
+    return {
+      range = range,
+      hl_group = hl_group,
+      priority = priority,
+      conceal = conceal,
+      spell = spell,
+      url = url,
+    },
+      node
+  end
+end
+
 ---@param self vim.treesitter.highlighter
 ---@param buf integer
 ---@param line integer
@@ -302,47 +370,37 @@ local function on_line_impl(self, buf, line, is_spell_nav)
 
       -- TODO(lewis6991): Creating a new iterator loses the cached predicate results for query
       -- matches. Move this logic inside iter_captures() so we can maintain the cache.
-      state.iter =
-        state.highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
+      state.iter = TSHighlighter._iter_highlight_info(
+        state.highlighter_query:query(),
+        root_node,
+        self.bufnr,
+        line,
+        root_end_row + 1
+      )
     end
 
     while line >= state.next_row do
-      local capture, node, metadata, match = state.iter(line)
+      local info, node = state.iter(line)
 
       local range = { root_end_row + 1, 0, root_end_row + 1, 0 }
       if node then
-        range = vim.treesitter.get_range(node, buf, metadata and metadata[capture])
+        range = info and info.range or vim.treesitter.get_range(node, buf)
       end
       local start_row, start_col, end_row, end_col = Range.unpack4(range)
 
-      if capture then
-        local hl = state.highlighter_query:get_hl_from_capture(capture)
+      if info then
+        local hl = state.highlighter_query:get_hl_from_capture(info.hl_group)
 
-        local capture_name = state.highlighter_query:query().captures[capture]
-
-        local spell, spell_pri_offset = get_spell(capture_name)
-
-        -- The "priority" attribute can be set at the pattern level or on a particular capture
-        local priority = (
-          tonumber(metadata.priority or metadata[capture] and metadata[capture].priority)
-          or vim.highlight.priorities.treesitter
-        ) + spell_pri_offset
-
-        -- The "conceal" attribute can be set at the pattern level or on a particular capture
-        local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
-
-        local url = get_url(match, buf, capture, metadata)
-
-        if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
+        if hl and end_row >= line and (not is_spell_nav or info.spell ~= nil) then
           api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
             end_line = end_row,
             end_col = end_col,
             hl_group = hl,
             ephemeral = true,
-            priority = priority,
-            conceal = conceal,
-            spell = spell,
-            url = url,
+            priority = info.priority,
+            conceal = info.conceal,
+            spell = info.spell,
+            url = info.url,
           })
         end
       end

src/nvim/buffer_defs.h

@@ -47,8 +47,8 @@ typedef struct {
 #define VALID_VIRTCOL   0x04    // w_virtcol (file col) is valid
 #define VALID_CHEIGHT   0x08    // w_cline_height and w_cline_folded valid
 #define VALID_CROW      0x10    // w_cline_row is valid
-#define VALID_BOTLINE   0x20    // w_botine and w_empty_rows are valid
-#define VALID_BOTLINE_AP 0x40   // w_botine is approximated
+#define VALID_BOTLINE   0x20    // w_botline and w_empty_rows are valid
+#define VALID_BOTLINE_AP 0x40   // w_botline is approximated
 #define VALID_TOPLINE   0x80    // w_topline is valid (for cursor position)
 
 // flags for b_flags

src/nvim/eval.lua

@@ -11621,6 +11621,10 @@ M.funcs = {
       	synconcealed(lnum, 4)   [1, 'X', 2]
       	synconcealed(lnum, 5)   [1, 'X', 2]
       	synconcealed(lnum, 6)   [0, '', 0]
+
+      Note: Doesn't consider |matchadd()| highlighting items,
+      since syntax and matching highlighting are two different
+      mechanisms |syntax-vs-match|.
     ]=],
     name = 'synconcealed',
     params = { { 'lnum', 'integer' }, { 'col', 'integer' } },

src/nvim/window.c

@@ -2467,6 +2467,7 @@ void win_init_empty(win_T *wp)
   wp->w_topline = 1;
   wp->w_topfill = 0;
   wp->w_botline = 2;
+  wp->w_valid = 0;
   wp->w_s = &wp->w_buffer->b_s;
 }
 

test/functional/treesitter/highlight_spec.lua

@@ -8,6 +8,7 @@ local exec_lua = n.exec_lua
 local feed = n.feed
 local command = n.command
 local api = n.api
+local exec = n.exec
 local eq = t.eq
 
 before_each(clear)
@@ -1044,3 +1045,69 @@ describe('treesitter highlighting (markdown)', function()
     }
   end)
 end)
+
+describe('LanguageTree:get_highlight()', function()
+  --- @type test.functional.ui.screen
+  local screen
+  before_each(function()
+    clear({ args = { '--clean' } })
+    screen = Screen.new(80, 80)
+    screen:attach({ term_name = 'xterm' })
+    exec('colorscheme default')
+  end)
+
+  local function run_assert()
+    exec_lua('vim.treesitter.start()')
+    exec('norm! ggO#;')
+    screen:expect({ any = vim.pesc('#^;') })
+    exec('norm! :\rh')
+    screen:expect({ any = vim.pesc('^#;') })
+    local expected = screen:get_snapshot()
+
+    local ns = api.nvim_create_namespace 'test-namespace'
+    api.nvim_buf_clear_namespace(0, ns, 0, -1)
+    for _, result in
+      ipairs(exec_lua [[
+      local parser = vim.treesitter.get_parser()
+      parser:parse(true)
+      return vim.treesitter.get_highlight(parser)
+    ]]) --[[@as fun():number,table]]
+    do
+      api.nvim_buf_set_extmark(0, ns, result.range[1], result.range[2], {
+        end_row = result.range[3],
+        end_col = result.range[4],
+        hl_group = result.hl_group,
+        priority = result.priority,
+      })
+    end
+
+    exec_lua('vim.treesitter.stop()')
+    exec('syntax off')
+    exec('norm! gg0f;')
+    screen:expect({ any = vim.pesc('#^;') })
+    exec('norm! :\rh')
+    screen:expect({ grid = expected.grid, attr_ids = expected.attr_ids })
+  end
+
+  it('works', function()
+    insert [[
+    function main()
+      print("hello world")
+    end
+    ]]
+
+    exec('setf lua')
+    run_assert()
+  end)
+
+  it('works with injected languages', function()
+    insert [[
+    lua << EOF
+    print("hello world")
+    EOF
+    ]]
+
+    exec('setf vim')
+    run_assert()
+  end)
+end)

test/old/testdir/test_autocmd.vim

@@ -4105,4 +4105,16 @@ func Test_SwapExists_set_other_buf_modified()
   bwipe!
 endfunc
 
+func Test_BufEnter_botline()
+  set hidden
+  call writefile(range(10), 'Xxx1', 'D')
+  call writefile(range(20), 'Xxx2', 'D')
+  edit Xxx1
+  edit Xxx2
+  au BufEnter Xxx1 call assert_true(line('w$') > 1)
+  edit Xxx1
+  au! BufEnter Xxx1
+  set hidden&vim
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab