Merge 2550ffcd5f into 435dee74bb

Problem:  When :edit an existing buffer, line('w$') may return a
          wrong result.
Solution: Reset w_valid in curwin_init() (Jaehwang Jung)

`do_ecmd()` reinitializes the current window (`curwin_init()`) whose
`w_valid` field may have `VALID_BOTLINE` set. Resetting `w_botline`
without marking it as invalid makes subsequent `validate_botline()`
calls a no-op, thus resulting in wrong `line('w$')` value.

closes: vim/vim#14642

eb80b8304e

Co-authored-by: Jaehwang Jung <tomtomjhj@gmail.com>

runtime/doc/builtin.txt

@@ -8208,6 +8208,10 @@ synconcealed({lnum}, {col})                                     *synconcealed()*
 			synconcealed(lnum, 5)   [1, 'X', 2]
 			synconcealed(lnum, 6)   [0, '', 0]
 
+		Note: Doesn't consider |matchadd()| highlighting items,
+		since syntax and matching highlighting are two different
+		mechanisms |syntax-vs-match|.
+
 synstack({lnum}, {col})                                             *synstack()*
 		Return a |List|, which is the stack of syntax items at the
 		position {lnum} and {col} in the current window.  {lnum} is

runtime/doc/pattern.txt

@@ -1375,6 +1375,19 @@ Finally, these constructs are unique to Perl:
 ==============================================================================
 10. Highlighting matches				*match-highlight*
 
+							*syntax-vs-match*
+		Note that the match highlight mechanism is independent
+		of |syntax-highlighting|, which is (usually) a buffer-local
+		highlighting, while matching is window-local, both methods
+		can be freely mixed.  Match highlighting functions give you
+		a bit more flexibility in when and how to apply, but are
+		typically only used for temporary highlighting, without strict
+		rules.  Both methods can be used to conceal text.
+
+		Thus the matching functions like |matchadd()| won't consider
+		syntax rules and functions like |synconcealed()| and the
+		other way around.
+
 							*:mat* *:match*
 :mat[ch] {group} /{pattern}/
 		Define a pattern to highlight in the current window.  It will

runtime/doc/syntax.txt

@@ -3825,7 +3825,9 @@ Whether or not it is actually concealed depends on the value of the
 'conceallevel' option.  The 'concealcursor' option is used to decide whether
 concealable items in the current line are displayed unconcealed to be able to
 edit the line.
-Another way to conceal text is with |matchadd()|.
+
+Another way to conceal text is with |matchadd()|, but internally this works a
+bit differently |syntax-vs-match|.
 
 concealends						*:syn-concealends*
 
@@ -3833,7 +3835,9 @@ When the "concealends" argument is given, the start and end matches of
 the region, but not the contents of the region, are marked as concealable.
 Whether or not they are actually concealed depends on the setting on the
 'conceallevel' option. The ends of a region can only be concealed separately
-in this way when they have their own highlighting via "matchgroup"
+in this way when they have their own highlighting via "matchgroup".  The
+|synconcealed()| function can be used to retrieve information about conealed
+items.
 
 cchar							*:syn-cchar*
 							*E844*

runtime/doc/usr_41.txt

@@ -948,7 +948,7 @@ Syntax and highlighting:	  *syntax-functions* *highlighting-functions*
 	synIDattr()		get a specific attribute of a syntax ID
 	synIDtrans()		get translated syntax ID
 	synstack()		get list of syntax IDs at a specific position
-	synconcealed()		get info about concealing
+	synconcealed()		get info about (syntax) concealing
 	diff_hlID()		get highlight ID for diff mode at a position
 	matchadd()		define a pattern to highlight (a "match")
 	matchaddpos()		define a list of positions to highlight

runtime/lua/vim/_meta/vimfn.lua

@@ -9752,6 +9752,10 @@ function vim.fn.synIDtrans(synID) end
 ---   synconcealed(lnum, 5)   [1, 'X', 2]
 ---   synconcealed(lnum, 6)   [0, '', 0]
 ---
+--- Note: Doesn't consider |matchadd()| highlighting items,
+--- since syntax and matching highlighting are two different
+--- mechanisms |syntax-vs-match|.
+---
 --- @param lnum integer
 --- @param col integer
 --- @return {[1]: integer, [2]: string, [3]: integer}

src/nvim/buffer_defs.h

@@ -47,8 +47,8 @@ typedef struct {
 #define VALID_VIRTCOL   0x04    // w_virtcol (file col) is valid
 #define VALID_CHEIGHT   0x08    // w_cline_height and w_cline_folded valid
 #define VALID_CROW      0x10    // w_cline_row is valid
-#define VALID_BOTLINE   0x20    // w_botine and w_empty_rows are valid
-#define VALID_BOTLINE_AP 0x40   // w_botine is approximated
+#define VALID_BOTLINE   0x20    // w_botline and w_empty_rows are valid
+#define VALID_BOTLINE_AP 0x40   // w_botline is approximated
 #define VALID_TOPLINE   0x80    // w_topline is valid (for cursor position)
 
 // flags for b_flags

src/nvim/eval.lua

@@ -11621,6 +11621,10 @@ M.funcs = {
       	synconcealed(lnum, 4)   [1, 'X', 2]
       	synconcealed(lnum, 5)   [1, 'X', 2]
       	synconcealed(lnum, 6)   [0, '', 0]
+
+      Note: Doesn't consider |matchadd()| highlighting items,
+      since syntax and matching highlighting are two different
+      mechanisms |syntax-vs-match|.
     ]=],
     name = 'synconcealed',
     params = { { 'lnum', 'integer' }, { 'col', 'integer' } },

src/nvim/lua/executor.c

@@ -1487,19 +1487,12 @@ static void nlua_typval_exec(const char *lcmd, size_t lcmd_len, const char *name
   }
 }
 
-void nlua_source_str(const char *code, char *name)
+void nlua_exec_ga(garray_T *ga, char *name)
 {
-  const sctx_T save_current_sctx = current_sctx;
-  current_sctx.sc_sid = SID_STR;
-  current_sctx.sc_seq = 0;
-  current_sctx.sc_lnum = 0;
-  estack_push(ETYPE_SCRIPT, name, 0);
-
+  char *code = ga_concat_strings_sep(ga, "\n");
   size_t len = strlen(code);
   nlua_typval_exec(code, len, name, NULL, 0, false, NULL);
-
-  estack_pop();
-  current_sctx = save_current_sctx;
+  xfree(code);
 }
 
 /// Call a LuaCallable given some typvals

src/nvim/lua/executor.h

@@ -8,6 +8,7 @@
 #include "nvim/cmdexpand_defs.h"  // IWYU pragma: keep
 #include "nvim/ex_cmds_defs.h"  // IWYU pragma: keep
 #include "nvim/func_attr.h"
+#include "nvim/garray_defs.h"  // IWYU pragma: keep
 #include "nvim/macros_defs.h"
 #include "nvim/types_defs.h"
 #include "nvim/usercmd.h"  // IWYU pragma: keep

src/nvim/runtime.c

@@ -73,7 +73,11 @@ typedef struct {
   FILE *fp;                     ///< opened file for sourcing
   char *nextline;               ///< if not NULL: line that was read ahead
   linenr_T sourcing_lnum;       ///< line number of the source file
-  int finished;                 ///< ":finish" used
+  bool source_str_from_lua;     ///< true if sourcing a string from Lua
+  bool finished;                ///< ":finish" used
+  bool source_from_buf;         ///< true if sourcing from a buffer or string
+  int buf_lnum;                 ///< line number in the buffer or string
+  garray_T buflines;            ///< lines in the buffer or string
 #ifdef USE_CRNL
   int fileformat;               ///< EOL_UNKNOWN, EOL_UNIX or EOL_DOS
   bool error;                   ///< true if LF found after CR-LF
@@ -1888,42 +1892,6 @@ static bool concat_continued_line(garray_T *const ga, const int init_growsize, c
   return true;
 }
 
-typedef struct {
-  char *buf;
-  size_t offset;
-} GetStrLineCookie;
-
-/// Get one full line from a sourced string (in-memory, no file).
-/// Called by do_cmdline() when it's called from do_source_str().
-///
-/// @return pointer to allocated line, or NULL for end-of-file or
-///         some error.
-static char *get_str_line(int c, void *cookie, int indent, bool do_concat)
-{
-  GetStrLineCookie *p = cookie;
-  if (strlen(p->buf) <= p->offset) {
-    return NULL;
-  }
-  const char *line = p->buf + p->offset;
-  const char *eol = skip_to_newline(line);
-  garray_T ga;
-  ga_init(&ga, sizeof(char), 400);
-  ga_concat_len(&ga, line, (size_t)(eol - line));
-  if (do_concat && vim_strchr(p_cpo, CPO_CONCAT) == NULL) {
-    while (eol[0] != NUL) {
-      line = eol + 1;
-      const char *const next_eol = skip_to_newline(line);
-      if (!concat_continued_line(&ga, 400, line, (size_t)(next_eol - line))) {
-        break;
-      }
-      eol = next_eol;
-    }
-  }
-  ga_append(&ga, NUL);
-  p->offset = (size_t)(eol - p->buf) + 1;
-  return ga.ga_data;
-}
-
 /// Create a new script item and allocate script-local vars. @see new_script_vars
 ///
 /// @param  name  File name of the script. NULL for anonymous :source.
@@ -1953,84 +1921,70 @@ scriptitem_T *new_script_item(char *const name, scid_T *const sid_out)
   return SCRIPT_ITEM(sid);
 }
 
-static int source_using_linegetter(void *cookie, LineGetter fgetline, const char *traceback_name)
-{
-  char *save_sourcing_name = SOURCING_NAME;
-  linenr_T save_sourcing_lnum = SOURCING_LNUM;
-  char sourcing_name_buf[256];
-  char *sname;
-  if (save_sourcing_name == NULL) {
-    sname = (char *)traceback_name;
-  } else {
-    snprintf(sourcing_name_buf, sizeof(sourcing_name_buf),
-             "%s called at %s:%" PRIdLINENR, traceback_name, save_sourcing_name,
-             save_sourcing_lnum);
-    sname = sourcing_name_buf;
-  }
-  estack_push(ETYPE_SCRIPT, sname, 0);
-
-  const sctx_T save_current_sctx = current_sctx;
-  if (current_sctx.sc_sid != SID_LUA) {
-    current_sctx.sc_sid = SID_STR;
-  }
-  current_sctx.sc_seq = 0;
-  current_sctx.sc_lnum = save_sourcing_lnum;
-  funccal_entry_T entry;
-  save_funccal(&entry);
-  int retval = do_cmdline(NULL, fgetline, cookie,
-                          DOCMD_VERBOSE | DOCMD_NOWAIT | DOCMD_REPEAT);
-  estack_pop();
-  current_sctx = save_current_sctx;
-  restore_funccal();
-  return retval;
-}
-
 void cmd_source_buffer(const exarg_T *const eap, bool ex_lua)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (curbuf == NULL) {
-    return;
-  }
-  garray_T ga;
-  ga_init(&ga, sizeof(char), 400);
-  const linenr_T final_lnum = eap->line2;
-  // Copy the contents to be executed.
-  for (linenr_T curr_lnum = eap->line1; curr_lnum <= final_lnum; curr_lnum++) {
-    // Adjust growsize to current length to speed up concatenating many lines.
-    if (ga.ga_len > 400) {
-      ga_set_growsize(&ga, MIN(ga.ga_len, 8000));
-    }
-    ga_concat(&ga, ml_get(curr_lnum));
-    ga_append(&ga, NL);
-  }
-  ((char *)ga.ga_data)[ga.ga_len - 1] = NUL;
-  if (ex_lua || strequal(curbuf->b_p_ft, "lua")
-      || (curbuf->b_fname && path_with_extension(curbuf->b_fname, "lua"))) {
-    char *name = ex_lua ? ":{range}lua" : ":source (no file)";
-    nlua_source_str(ga.ga_data, name);
-  } else {
-    const GetStrLineCookie cookie = {
-      .buf = ga.ga_data,
-      .offset = 0,
-    };
-    source_using_linegetter((void *)&cookie, get_str_line, ":source (no file)");
-  }
-  ga_clear(&ga);
+  do_source_ext(NULL, false, DOSO_NONE, NULL, eap, ex_lua, NULL);
 }
 
-/// Executes lines in `src` as Ex commands.
-///
-/// @see do_source()
-int do_source_str(const char *cmd, const char *traceback_name)
+/// Initialization for sourcing lines from the current buffer. Reads all the
+/// lines from the buffer and stores it in the cookie grow array.
+/// Returns a pointer to the name ":source buffer=<n>" on success and NULL on failure.
+static char *do_source_buffer_init(source_cookie_T *sp, const exarg_T *eap, bool ex_lua)
+  FUNC_ATTR_NONNULL_ALL
 {
-  GetStrLineCookie cookie = {
-    .buf = (char *)cmd,
-    .offset = 0,
-  };
-  return source_using_linegetter((void *)&cookie, get_str_line, traceback_name);
+  if (curbuf == NULL) {
+    return NULL;
+  }
+
+  if (ex_lua) {
+    // Use ":{range}lua buffer=<num>" as the script name
+    snprintf(IObuff, IOSIZE, ":{range}lua buffer=%d", curbuf->b_fnum);
+  } else {
+    // Use ":source buffer=<num>" as the script name
+    snprintf(IObuff, IOSIZE, ":source buffer=%d", curbuf->b_fnum);
+  }
+  char *fname = xstrdup(IObuff);
+
+  ga_init(&sp->buflines, sizeof(char *), 100);
+  // Copy the lines from the buffer into a grow array
+  for (linenr_T curr_lnum = eap->line1; curr_lnum <= eap->line2; curr_lnum++) {
+    GA_APPEND(char *, &sp->buflines, xstrdup(ml_get(curr_lnum)));
+  }
+  sp->buf_lnum = 0;
+  sp->source_from_buf = true;
+  // When sourcing a range of lines from a buffer, use buffer line number.
+  sp->sourcing_lnum = eap->line1 - 1;
+
+  return fname;
 }
 
-/// When fname is a 'lua' file nlua_exec_file() is invoked to source it.
+/// Initialization for sourcing lines from a string. Reads all the
+/// lines from the string and stores it in the cookie grow array.
+static void do_source_str_init(source_cookie_T *sp, const char *cmd)
+  FUNC_ATTR_NONNULL_ALL
+{
+  ga_init(&sp->buflines, sizeof(char *), 100);
+  // Copy the lines from the string into a grow array
+  while (*cmd != NUL) {
+    const char *eol = skip_to_newline(cmd);
+    GA_APPEND(char *, &sp->buflines, xmemdupz(cmd, (size_t)(eol - cmd)));
+    cmd = eol + (*eol != NUL);
+  }
+  sp->source_str_from_lua = current_sctx.sc_sid == SID_LUA;
+  sp->buf_lnum = 0;
+  sp->source_from_buf = true;
+}
+
+/// Executes lines in `cmd` as Ex commands.
+///
+/// @see do_source_ext()
+int do_source_str(const char *cmd, char *traceback_name)
+{
+  return do_source_ext(traceback_name, false, DOSO_NONE, NULL, NULL, false, cmd);
+}
+
+/// When fname is a .lua file nlua_exec_file() is invoked to source it.
 /// Otherwise reads the file `fname` and executes its lines as Ex commands.
 ///
 /// This function may be called recursively!
@@ -2041,11 +1995,14 @@ int do_source_str(const char *cmd, const char *traceback_name)
 /// @param check_other  check for .vimrc and _vimrc
 /// @param is_vimrc     DOSO_ value
 /// @param ret_sid      if not NULL and we loaded the script before, don't load it again
+/// @param eap          used when sourcing lines from a buffer instead of a file
+/// @param cmd          if not NULL, source from the given string
 ///
 /// @return  FAIL if file could not be opened, OK otherwise
 ///
 /// If a scriptitem_T was found or created "*ret_sid" is set to the SID.
-int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
+static int do_source_ext(char *fname, bool check_other, int is_vimrc, int *ret_sid,
+                         const exarg_T *eap, bool ex_lua, const char *cmd)
 {
   source_cookie_T cookie;
   uint8_t *firstline = NULL;
@@ -2055,22 +2012,35 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   proftime_T wait_start;
   bool trigger_source_post = false;
 
-  char *p = expand_env_save(fname);
-  if (p == NULL) {
-    return retval;
-  }
-  char *fname_exp = fix_fname(p);
-  xfree(p);
-  if (fname_exp == NULL) {
-    return retval;
-  }
-  if (os_isdir(fname_exp)) {
-    smsg(0, _("Cannot source a directory: \"%s\""), fname);
-    goto theend;
+  CLEAR_FIELD(cookie);
+  char *fname_exp = NULL;
+  if (fname == NULL) {
+    // sourcing lines from a buffer
+    fname_exp = do_source_buffer_init(&cookie, eap, ex_lua);
+    if (fname_exp == NULL) {
+      return FAIL;
+    }
+  } else if (cmd != NULL) {
+    do_source_str_init(&cookie, cmd);
+    fname_exp = xstrdup(fname);
+  } else {
+    char *p = expand_env_save(fname);
+    if (p == NULL) {
+      return retval;
+    }
+    fname_exp = fix_fname(p);
+    xfree(p);
+    if (fname_exp == NULL) {
+      return retval;
+    }
+    if (os_isdir(fname_exp)) {
+      smsg(0, _("Cannot source a directory: \"%s\""), fname);
+      goto theend;
+    }
   }
 
   // See if we loaded this script before.
-  int sid = find_script_by_name(fname_exp);
+  int sid = cmd != NULL ? SID_STR : find_script_by_name(fname_exp);
   if (sid > 0 && ret_sid != NULL) {
     // Already loaded and no need to load again, return here.
     *ret_sid = sid;
@@ -2093,11 +2063,13 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   // Apply SourcePre autocommands, they may get the file.
   apply_autocmds(EVENT_SOURCEPRE, fname_exp, fname_exp, false, curbuf);
 
-  cookie.fp = fopen_noinh_readbin(fname_exp);
+  if (!cookie.source_from_buf) {
+    cookie.fp = fopen_noinh_readbin(fname_exp);
+  }
   if (cookie.fp == NULL && check_other) {
     // Try again, replacing file name ".nvimrc" by "_nvimrc" or vice versa,
     // and ".exrc" by "_exrc" or vice versa.
-    p = path_tail(fname_exp);
+    char *p = path_tail(fname_exp);
     if ((*p == '.' || *p == '_')
         && (STRICMP(p + 1, "nvimrc") == 0 || STRICMP(p + 1, "exrc") == 0)) {
       *p = (*p == '_') ? '.' : '_';
@@ -2105,7 +2077,7 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
     }
   }
 
-  if (cookie.fp == NULL) {
+  if (cookie.fp == NULL && !cookie.source_from_buf) {
     if (p_verbose > 1) {
       verbose_enter();
       if (SOURCING_NAME == NULL) {
@@ -2142,13 +2114,8 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   } else {
     cookie.fileformat = EOL_UNKNOWN;
   }
-  cookie.error = false;
 #endif
 
-  cookie.nextline = NULL;
-  cookie.sourcing_lnum = 0;
-  cookie.finished = false;
-
   // Check if this script has a breakpoint.
   cookie.breakpoint = dbg_find_breakpoint(true, fname_exp, 0);
   cookie.fname = fname_exp;
@@ -2177,7 +2144,9 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
 
   const sctx_T save_current_sctx = current_sctx;
 
-  current_sctx.sc_lnum = 0;
+  if (!cookie.source_str_from_lua) {
+    current_sctx.sc_lnum = 0;
+  }
 
   // Always use a new sequence number.
   current_sctx.sc_seq = ++last_current_SID_seq;
@@ -2185,7 +2154,7 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   if (sid > 0) {
     // loading the same script again
     si = SCRIPT_ITEM(sid);
-  } else {
+  } else if (cmd == NULL) {
     // It's new, generate a new SID.
     si = new_script_item(fname_exp, &sid);
     fname_exp = xstrdup(si->sn_name);  // used for autocmd
@@ -2193,12 +2162,14 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
       *ret_sid = sid;
     }
   }
-  current_sctx.sc_sid = sid;
+  if (!cookie.source_str_from_lua) {
+    current_sctx.sc_sid = sid;
+  }
 
   // Keep the sourcing name/lnum, for recursive calls.
-  estack_push(ETYPE_SCRIPT, si->sn_name, 0);
+  estack_push(ETYPE_SCRIPT, si != NULL ? si->sn_name : fname_exp, 0);
 
-  if (l_do_profiling == PROF_YES) {
+  if (l_do_profiling == PROF_YES && si != NULL) {
     bool forceit = false;
 
     // Check if we do profiling for this script.
@@ -2215,7 +2186,11 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
 
   cookie.conv.vc_type = CONV_NONE;              // no conversion
 
-  if (path_with_extension(fname_exp, "lua")) {
+  if (fname == NULL
+      && (ex_lua || strequal(curbuf->b_p_ft, "lua")
+          || (curbuf->b_fname && path_with_extension(curbuf->b_fname, "lua")))) {
+    nlua_exec_ga(&cookie.buflines, fname_exp);
+  } else if (path_with_extension(fname_exp, "lua")) {
     const sctx_T current_sctx_backup = current_sctx;
     current_sctx.sc_sid = SID_LUA;
     current_sctx.sc_lnum = 0;
@@ -2229,7 +2204,7 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
         && firstline[1] == 0xbb && firstline[2] == 0xbf) {
       // Found BOM; setup conversion, skip over BOM and recode the line.
       convert_setup(&cookie.conv, "utf-8", p_enc);
-      p = string_convert(&cookie.conv, (char *)firstline + 3, NULL);
+      char *p = string_convert(&cookie.conv, (char *)firstline + 3, NULL);
       if (p == NULL) {
         p = xstrdup((char *)firstline + 3);
       }
@@ -2242,7 +2217,7 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   }
   retval = OK;
 
-  if (l_do_profiling == PROF_YES) {
+  if (l_do_profiling == PROF_YES && si != NULL) {
     // Get "si" again, "script_items" may have been reallocated.
     si = SCRIPT_ITEM(current_sctx.sc_sid);
     if (si->sn_prof_on) {
@@ -2289,7 +2264,12 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   if (l_do_profiling == PROF_YES) {
     prof_child_exit(&wait_start);    // leaving a child now
   }
-  fclose(cookie.fp);
+  if (cookie.fp != NULL) {
+    fclose(cookie.fp);
+  }
+  if (cookie.source_from_buf) {
+    ga_clear_strings(&cookie.buflines);
+  }
   xfree(cookie.nextline);
   xfree(firstline);
   convert_setup(&cookie.conv, NULL, NULL);
@@ -2303,6 +2283,13 @@ theend:
   return retval;
 }
 
+/// @param check_other  check for .vimrc and _vimrc
+/// @param is_vimrc     DOSO_ value
+int do_source(char *fname, bool check_other, int is_vimrc, int *ret_sid)
+{
+  return do_source_ext(fname, check_other, is_vimrc, ret_sid, NULL, false, NULL);
+}
+
 /// Find an already loaded script "name".
 /// If found returns its script ID.  If not found returns -1.
 int find_script_by_name(char *name)
@@ -2545,7 +2532,7 @@ char *getsourceline(int c, void *cookie, int indent, bool do_concat)
   char *line;
 
   // If breakpoints have been added/deleted need to check for it.
-  if (sp->dbg_tick < debug_tick) {
+  if ((sp->dbg_tick < debug_tick) && !sp->source_from_buf) {
     sp->breakpoint = dbg_find_breakpoint(true, sp->fname, SOURCING_LNUM);
     sp->dbg_tick = debug_tick;
   }
@@ -2556,7 +2543,7 @@ char *getsourceline(int c, void *cookie, int indent, bool do_concat)
   SOURCING_LNUM = sp->sourcing_lnum + 1;
   // Get current line.  If there is a read-ahead line, use it, otherwise get
   // one now.  "fp" is NULL if actually using a string.
-  if (sp->finished || sp->fp == NULL) {
+  if (sp->finished || (!sp->source_from_buf && sp->fp == NULL)) {
     line = NULL;
   } else if (sp->nextline == NULL) {
     line = get_one_sourceline(sp);
@@ -2609,7 +2596,7 @@ char *getsourceline(int c, void *cookie, int indent, bool do_concat)
   }
 
   // Did we encounter a breakpoint?
-  if (sp->breakpoint != 0 && sp->breakpoint <= SOURCING_LNUM) {
+  if (!sp->source_from_buf && sp->breakpoint != 0 && sp->breakpoint <= SOURCING_LNUM) {
     dbg_breakpoint(sp->fname, SOURCING_LNUM);
     // Find next breakpoint.
     sp->breakpoint = dbg_find_breakpoint(true, sp->fname, SOURCING_LNUM);
@@ -2638,19 +2625,28 @@ static char *get_one_sourceline(source_cookie_T *sp)
   while (true) {
     // make room to read at least 120 (more) characters
     ga_grow(&ga, 120);
-    buf = ga.ga_data;
-
-retry:
-    errno = 0;
-    if (fgets(buf + ga.ga_len, ga.ga_maxlen - ga.ga_len,
-              sp->fp) == NULL) {
-      if (errno == EINTR) {
-        goto retry;
+    if (sp->source_from_buf) {
+      if (sp->buf_lnum >= sp->buflines.ga_len) {
+        break;              // all the lines are processed
       }
-
-      break;
+      ga_concat(&ga, ((char **)sp->buflines.ga_data)[sp->buf_lnum]);
+      sp->buf_lnum++;
+      ga_grow(&ga, 1);
+      buf = (char *)ga.ga_data;
+      buf[ga.ga_len++] = NUL;
+      len = ga.ga_len;
+    } else {
+      buf = ga.ga_data;
+retry:
+      errno = 0;
+      if (fgets(buf + ga.ga_len, ga.ga_maxlen - ga.ga_len, sp->fp) == NULL) {
+        if (errno == EINTR) {
+          goto retry;
+        }
+        break;
+      }
+      len = ga.ga_len + (int)strlen(buf + ga.ga_len);
     }
-    len = ga.ga_len + (int)strlen(buf + ga.ga_len);
 #ifdef USE_CRNL
     // Ignore a trailing CTRL-Z, when in Dos mode. Only recognize the
     // CTRL-Z by its own, or after a NL.

src/nvim/window.c

@@ -2467,6 +2467,7 @@ void win_init_empty(win_T *wp)
   wp->w_topline = 1;
   wp->w_topfill = 0;
   wp->w_botline = 2;
+  wp->w_valid = 0;
   wp->w_s = &wp->w_buffer->b_s;
 }
 

test/functional/ex_cmds/source_spec.lua

@@ -231,9 +231,9 @@ describe(':source', function()
 
         eq(12, eval('g:c'))
         eq('    \\ 1\n   "\\ 2', exec_lua('return _G.a'))
-        eq(':source (no file)', api.nvim_get_var('sfile_value'))
-        eq(':source (no file)', api.nvim_get_var('stack_value'))
-        eq(':source (no file)', api.nvim_get_var('script_value'))
+        eq(':source buffer=1', api.nvim_get_var('sfile_value'))
+        eq(':source buffer=1', api.nvim_get_var('stack_value'))
+        eq(':source buffer=1', api.nvim_get_var('script_value'))
       end)
     end
 

test/functional/lua/commands_spec.lua

@@ -207,7 +207,8 @@ describe(':lua', function()
 
     -- ":{range}lua" fails on invalid Lua code.
     eq(
-      [[:{range}lua: Vim(lua):E5107: Error loading lua [string ":{range}lua"]:0: '=' expected near '<eof>']],
+      [[:{range}lua buffer=1: Vim(lua):E5107: Error loading lua ]]
+        .. [[[string ":{range}lua buffer=1"]:0: '=' expected near '<eof>']],
       pcall_err(command, '1lua')
     )
 

test/old/testdir/test_autocmd.vim

@@ -4105,4 +4105,16 @@ func Test_SwapExists_set_other_buf_modified()
   bwipe!
 endfunc
 
+func Test_BufEnter_botline()
+  set hidden
+  call writefile(range(10), 'Xxx1', 'D')
+  call writefile(range(20), 'Xxx2', 'D')
+  edit Xxx1
+  edit Xxx2
+  au BufEnter Xxx1 call assert_true(line('w$') > 1)
+  edit Xxx1
+  au! BufEnter Xxx1
+  set hidden&vim
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

test/old/testdir/test_source.vim

@@ -110,4 +110,520 @@ func Test_nested_script()
   call StopVimInTerminal(buf)
 endfunc
 
+" Test for sourcing a script from the current buffer
+func Test_source_buffer()
+  new
+  " Source a simple script
+  let lines =<< trim END
+    let a = "Test"
+    let b = 20
+
+    let c = [1.1]
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(['Test', 20, [1.1]], [g:a, g:b, g:c])
+
+  " Source a range of lines in the current buffer
+  %d _
+  let lines =<< trim END
+    let a = 10
+    let a += 20
+    let a += 30
+    let a += 40
+  END
+  call setline(1, lines)
+  .source
+  call assert_equal(10, g:a)
+  3source
+  call assert_equal(40, g:a)
+  2,3source
+  call assert_equal(90, g:a)
+
+  " Make sure the script line number is correct when sourcing a range of
+  " lines.
+  %d _
+  let lines =<< trim END
+     Line 1
+     Line 2
+     func Xtestfunc()
+       return expand("<sflnum>")
+     endfunc
+     Line 3
+     Line 4
+  END
+  call setline(1, lines)
+  3,5source
+  call assert_equal('4', Xtestfunc())
+  delfunc Xtestfunc
+
+  " Source a script with line continuation lines
+  %d _
+  let lines =<< trim END
+    let m = [
+      \   1,
+      \   2,
+      \ ]
+    call add(m, 3)
+  END
+  call setline(1, lines)
+  source
+  call assert_equal([1, 2, 3], g:m)
+  " Source a script with line continuation lines and a comment
+  %d _
+  let lines =<< trim END
+    let m = [
+      "\ first entry
+      \   'a',
+      "\ second entry
+      \   'b',
+      \ ]
+    " third entry
+    call add(m, 'c')
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(['a', 'b', 'c'], g:m)
+  " Source an incomplete line continuation line
+  %d _
+  let lines =<< trim END
+    let k = [
+      \
+  END
+  call setline(1, lines)
+  call assert_fails('source', 'E697:')
+  " Source a function with a for loop
+  %d _
+  let lines =<< trim END
+    let m = []
+    " test function
+    func! Xtest()
+      for i in range(5, 7)
+        call add(g:m, i)
+      endfor
+    endfunc
+    call Xtest()
+  END
+  call setline(1, lines)
+  source
+  call assert_equal([5, 6, 7], g:m)
+  " Source an empty buffer
+  %d _
+  source
+
+  " test for script local functions and variables
+  let lines =<< trim END
+    let s:var1 = 10
+    func s:F1()
+      let s:var1 += 1
+      return s:var1
+    endfunc
+    func s:F2()
+    endfunc
+    let g:ScriptID = expand("<SID>")
+  END
+  call setline(1, lines)
+  source
+  call assert_true(g:ScriptID != '')
+  call assert_true(exists('*' .. g:ScriptID .. 'F1'))
+  call assert_true(exists('*' .. g:ScriptID .. 'F2'))
+  call assert_equal(11, call(g:ScriptID .. 'F1', []))
+
+  " the same script ID should be used even if the buffer is sourced more than
+  " once
+  %d _
+  let lines =<< trim END
+    let g:ScriptID = expand("<SID>")
+    let g:Count += 1
+  END
+  call setline(1, lines)
+  let g:Count = 0
+  source
+  call assert_true(g:ScriptID != '')
+  let scid = g:ScriptID
+  source
+  call assert_equal(scid, g:ScriptID)
+  call assert_equal(2, g:Count)
+  source
+  call assert_equal(scid, g:ScriptID)
+  call assert_equal(3, g:Count)
+
+  " test for the script line number
+  %d _
+  let lines =<< trim END
+    " comment
+    let g:Slnum1 = expand("<slnum>")
+    let i = 1 +
+           \ 2 +
+          "\ comment
+           \ 3
+    let g:Slnum2 = expand("<slnum>")
+  END
+  call setline(1, lines)
+  source
+  call assert_equal('2', g:Slnum1)
+  call assert_equal('7', g:Slnum2)
+
+  " test for retaining the same script number across source calls
+  let lines =<< trim END
+     let g:ScriptID1 = expand("<SID>")
+     let g:Slnum1 = expand("<slnum>")
+     let l =<< trim END
+       let g:Slnum2 = expand("<slnum>")
+       let g:ScriptID2 = expand("<SID>")
+     END
+     new
+     call setline(1, l)
+     source
+     bw!
+     let g:ScriptID3 = expand("<SID>")
+     let g:Slnum3 = expand("<slnum>")
+  END
+  call writefile(lines, 'Xscript')
+  source Xscript
+  call assert_true(g:ScriptID1 != g:ScriptID2)
+  call assert_equal(g:ScriptID1, g:ScriptID3)
+  call assert_equal('2', g:Slnum1)
+  call assert_equal('1', g:Slnum2)
+  call assert_equal('12', g:Slnum3)
+  call delete('Xscript')
+
+  " test for sourcing a heredoc
+  %d _
+  let lines =<< trim END
+     let a = 1
+     let heredoc =<< trim DATA
+        red
+          green
+        blue
+     DATA
+     let b = 2
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(['red', '  green', 'blue'], g:heredoc)
+
+  " test for a while and for statement
+  %d _
+  let lines =<< trim END
+     let a = 0
+     let b = 1
+     while b <= 10
+       let a += 10
+       let b += 1
+     endwhile
+     for i in range(5)
+       let a += 10
+     endfor
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(150, g:a)
+
+  " test for sourcing the same buffer multiple times after changing a function
+  %d _
+  let lines =<< trim END
+     func Xtestfunc()
+       return "one"
+     endfunc
+  END
+  call setline(1, lines)
+  source
+  call assert_equal("one", Xtestfunc())
+  call setline(2, '  return "two"')
+  source
+  call assert_equal("two", Xtestfunc())
+  call setline(2, '  return "three"')
+  source
+  call assert_equal("three", Xtestfunc())
+  delfunc Xtestfunc
+
+  " test for using try/catch
+  %d _
+  let lines =<< trim END
+     let Trace = '1'
+     try
+       let a1 = b1
+     catch
+       let Trace ..= '2'
+     finally
+       let Trace ..= '3'
+     endtry
+  END
+  call setline(1, lines)
+  source
+  call assert_equal("123", g:Trace)
+
+  " test with the finish command
+  %d _
+  let lines =<< trim END
+     let g:Color = 'blue'
+     finish
+     let g:Color = 'green'
+  END
+  call setline(1, lines)
+  source
+  call assert_equal('blue', g:Color)
+
+  " Test for the SourcePre and SourcePost autocmds
+  augroup Xtest
+    au!
+    au SourcePre * let g:XsourcePre=4
+          \ | let g:XsourcePreFile = expand("<afile>")
+    au SourcePost * let g:XsourcePost=6
+          \ | let g:XsourcePostFile = expand("<afile>")
+  augroup END
+  %d _
+  let lines =<< trim END
+     let a = 1
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(4, g:XsourcePre)
+  call assert_equal(6, g:XsourcePost)
+  call assert_equal(':source buffer=' .. bufnr(), g:XsourcePreFile)
+  call assert_equal(':source buffer=' .. bufnr(), g:XsourcePostFile)
+  augroup Xtest
+    au!
+  augroup END
+  augroup! Xtest
+
+  %bw!
+endfunc
+
+" Test for sourcing a Vim9 script from the current buffer
+func Test_source_buffer_vim9()
+  throw 'Skipped: Vim9 script is N/A'
+  new
+
+  " test for sourcing a Vim9 script
+  %d _
+  let lines =<< trim END
+     vim9script
+
+     # check dict
+     var x: number = 10
+     def g:Xtestfunc(): number
+       return x
+     enddef
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(10, Xtestfunc())
+
+  " test for sourcing a vim9 script with line continuation
+  %d _
+  let lines =<< trim END
+     vim9script
+
+     g:Str1 = "hello "
+              .. "world"
+              .. ", how are you?"
+     g:Colors = [
+       'red',
+       # comment
+       'blue'
+       ]
+     g:Dict = {
+       a: 22,
+       # comment
+       b: 33
+       }
+
+     # calling a function with line continuation
+     def Sum(...values: list<number>): number
+       var sum: number = 0
+       for v in values
+         sum += v
+       endfor
+       return sum
+     enddef
+     g:Total1 = Sum(10,
+                   20,
+                   30)
+
+     var i: number = 0
+     while i < 10
+       # while loop
+       i +=
+           1
+     endwhile
+     g:Count1 = i
+
+     # for loop
+     g:Count2 = 0
+     for j in range(10, 20)
+       g:Count2 +=
+           i
+     endfor
+
+     g:Total2 = 10 +
+                20 -
+                5
+
+     g:Result1 = g:Total2 > 1
+                ? 'red'
+                : 'blue'
+
+     g:Str2 = 'x'
+              ->repeat(10)
+              ->trim()
+              ->strpart(4)
+
+     g:Result2 = g:Dict
+                    .a
+
+     augroup Test
+       au!
+       au BufNewFile Xfile g:readFile = 1
+             | g:readExtra = 2
+     augroup END
+     g:readFile = 0
+     g:readExtra = 0
+     new Xfile
+     bwipe!
+     augroup Test
+       au!
+     augroup END
+  END
+  call setline(1, lines)
+  source
+  call assert_equal("hello world, how are you?", g:Str1)
+  call assert_equal(['red', 'blue'], g:Colors)
+  call assert_equal(#{a: 22, b: 33}, g:Dict)
+  call assert_equal(60, g:Total1)
+  call assert_equal(10, g:Count1)
+  call assert_equal(110, g:Count2)
+  call assert_equal(25, g:Total2)
+  call assert_equal('red', g:Result1)
+  call assert_equal('xxxxxx', g:Str2)
+  call assert_equal(22, g:Result2)
+  call assert_equal(1, g:readFile)
+  call assert_equal(2, g:readExtra)
+
+  " test for sourcing the same buffer multiple times after changing a function
+  %d _
+  let lines =<< trim END
+     vim9script
+     def g:Xtestfunc(): string
+       return "one"
+     enddef
+  END
+  call setline(1, lines)
+  source
+  call assert_equal("one", Xtestfunc())
+  call setline(3, '  return "two"')
+  source
+  call assert_equal("two", Xtestfunc())
+  call setline(3, '  return "three"')
+  source
+  call assert_equal("three", Xtestfunc())
+  delfunc Xtestfunc
+
+  " Test for sourcing a range of lines. Make sure the script line number is
+  " correct.
+  %d _
+  let lines =<< trim END
+     Line 1
+     Line 2
+     vim9script
+     def g:Xtestfunc(): string
+       return expand("<sflnum>")
+     enddef
+     Line 3
+     Line 4
+  END
+  call setline(1, lines)
+  3,6source
+  call assert_equal('5', Xtestfunc())
+  delfunc Xtestfunc
+
+  " test for sourcing a heredoc
+  %d _
+  let lines =<< trim END
+    vim9script
+    var a = 1
+    g:heredoc =<< trim DATA
+       red
+         green
+       blue
+    DATA
+    var b = 2
+  END
+  call setline(1, lines)
+  source
+  call assert_equal(['red', '  green', 'blue'], g:heredoc)
+
+  " test for using the :vim9cmd modifier
+  %d _
+  let lines =<< trim END
+    first line
+    g:Math = {
+         pi: 3.12,
+         e: 2.71828
+      }
+    g:Editors = [
+      'vim',
+      # comment
+      'nano'
+      ]
+    last line
+  END
+  call setline(1, lines)
+  vim9cmd :2,10source
+  call assert_equal(#{pi: 3.12, e: 2.71828}, g:Math)
+  call assert_equal(['vim', 'nano'], g:Editors)
+
+  " test for using try/catch
+  %d _
+  let lines =<< trim END
+     vim9script
+     g:Trace = '1'
+     try
+       a1 = b1
+     catch
+       g:Trace ..= '2'
+     finally
+       g:Trace ..= '3'
+     endtry
+  END
+  call setline(1, lines)
+  source
+  call assert_equal('123', g:Trace)
+
+  " test with the finish command
+  %d _
+  let lines =<< trim END
+     vim9script
+     g:Color = 'red'
+     finish
+     g:Color = 'blue'
+  END
+  call setline(1, lines)
+  source
+  call assert_equal('red', g:Color)
+
+  %bw!
+endfunc
+
+func Test_source_buffer_long_line()
+  " This was reading past the end of the line.
+  new
+  norm300gr0
+  so
+  bwipe!
+
+  let lines =<< trim END
+      new
+      norm 10a0000000000ø00000000000
+      norm i0000000000000000000
+      silent! so
+  END
+  call writefile(lines, 'Xtest.vim')
+  source Xtest.vim
+  bwipe!
+  call delete('Xtest.vim')
+endfunc
+
+
 " vim: shiftwidth=2 sts=2 expandtab