Merge 0e09423f4e into 435dee74bb

Problem:  When :edit an existing buffer, line('w$') may return a
          wrong result.
Solution: Reset w_valid in curwin_init() (Jaehwang Jung)

`do_ecmd()` reinitializes the current window (`curwin_init()`) whose
`w_valid` field may have `VALID_BOTLINE` set. Resetting `w_botline`
without marking it as invalid makes subsequent `validate_botline()`
calls a no-op, thus resulting in wrong `line('w$')` value.

closes: vim/vim#14642

eb80b8304e

Co-authored-by: Jaehwang Jung <tomtomjhj@gmail.com>

BRANCH_TODO.md

@@ -0,0 +1,45 @@
+Dependencies:
+
+- [x] libuv
+- [-] lua
+- [ ] luajit
+- [x] MSGPACK_URL https://github.com/msgpack/msgpack-c/archive/c-6.0.1.tar.gz
+- [x] UNIBILIUM_URL https://github.com/neovim/unibilium/archive/d72c3598e7ac5d1ebf86ee268b8b4ed95c0fa628.tar.gz
+- [ ] LIBVTERM_URL https://github.com/neovim/libvterm/archive/v0.3.3.tar.gz
+- [-] luv
+- [-] lpeg
+- [x] compat53 (c api part only - but likely enough)
+- [x] TREESITTER_URL https://github.com/tree-sitter/tree-sitter/archive/v0.22.5.tar.gz
+
+non-glibc:
+- [ ] GETTEXT_URL https://github.com/neovim/deps/raw/b9bf36eb31f27e8136d907da38fa23518927737e/opt/gettext-0.20.1.tar.gz
+- [ ] LIBICONV_URL https://github.com/neovim/deps/raw/b9bf36eb31f27e8136d907da38fa23518927737e/opt/libiconv-1.17.tar.gz
+
+Runtime dependencies:
+
+- [ ] treesitter parsers
+- [ ] cat/tee/xxd.exe
+- [ ] WIN32YANK_X86_64_URL https://github.com/equalsraf/win32yank/releases/download/v0.1.1/win32yank-x64.zip
+
+
+Generators:
+
+ - [ ] gen_api_dispatch.lua
+ - [ ] gen_api_ui_events.lua
+ - [x] gen_char_blob.lua (partial - LUAC_PRG not supported)
+ - [x] gen_declarations.lua (partial - some generators generate input for this!)
+ - [ ] gen_eval.lua
+ - [ ] gen_events.lua
+ - [x] gen_ex_cmds.lua
+ - [x] gen_options_enum.lua
+ - [x] gen_options.lua
+ - [ ] gen_unicode_tables.lua
+ - [ ] gen_vimvim.lua
+
+ """scripts"""
+
+- [ ] gen_eval_files.lua*
+- [ ] gen_vimdoc.lua*
+- [ ] gen_filetype.lua
+- [ ] gen_help_html.lua
+- [ ] gen_lsp.lua

build.zig

@@ -0,0 +1,356 @@
+const std = @import("std");
+const LazyPath = std.Build.LazyPath;
+
+pub fn build(b: *std.Build) !void {
+    // Standard target options allows the person running `zig build` to choose
+    // what target to build for. Here we do not override the defaults, which
+    // means any target is allowed, and the default is native. Other options
+    // for restricting supported target set are available.
+    const target = b.standardTargetOptions(.{});
+
+    // Standard optimization options allow the person running `zig build` to select
+    // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not
+    // set a preferred release mode, allowing the user to decide how to optimize.
+    const optimize = b.standardOptimizeOption(.{});
+
+    const use_luajit = false;
+
+    const ziglua = b.dependency("ziglua", .{
+        .target = target,
+        .optimize = optimize,
+        .lang = if (use_luajit) .luajit else .lua51,
+        .shared = false,
+    });
+
+    const lpeg = b.dependency("lpeg", .{});
+
+    const lua = ziglua.artifact("lua");
+
+    const libuv_dep = b.dependency("libuv", .{ .target = target, .optimize = optimize });
+    const libuv = libuv_dep.artifact("uv");
+
+    const libluv = try build_libluv(b, target, optimize, lua, libuv);
+
+    const msgpack_c = b.dependency("msgpack_c", .{ .target = target, .optimize = optimize });
+    const unibilium = b.dependency("unibilium", .{ .target = target, .optimize = optimize });
+    const treesitter = b.dependency("treesitter", .{ .target = target, .optimize = optimize });
+    const vterm = b.dependency("vterm", .{ .target = target, .optimize = optimize });
+
+    const nlua0 = build_nlua0(b, target, optimize, use_luajit, ziglua, libluv, lpeg);
+
+    const wip_step = b.step("wip", "rearrange the power of it all");
+
+    // TODO(zig): using getEmittedIncludeTree() is ugly af. we want run_preprocessor()
+    // to use the std.build.Module include_path thing
+    const include_path = &.{
+        b.path("src/"),
+        b.path("src/includes_fixmelater/"),
+        lua.getEmittedIncludeTree(),
+        libuv.getEmittedIncludeTree(),
+        libluv.getEmittedIncludeTree(),
+        msgpack_c.artifact("msgpack_c").getEmittedIncludeTree(),
+        unibilium.artifact("unibilium").getEmittedIncludeTree(),
+        treesitter.artifact("tree-sitter").getEmittedIncludeTree(),
+        vterm.artifact("vterm").getEmittedIncludeTree(),
+    };
+
+    // usual caveat emptor: might need to force a rebuild if the only change is
+    // addition of new .c files, as those are not seen by any hash
+    const subdirs = [_][]const u8{
+        ".", // src/nvim itself
+        "os",
+        "api",
+        "api/private",
+        "msgpack_rpc",
+        "tui",
+        "event",
+        "eval",
+        "lua",
+        "viml",
+        "viml/parser",
+    };
+
+    // source names _relative_ src/nvim/, not including other src/ subdircs
+    var nvim_sources = try std.ArrayList([]u8).initCapacity(b.allocator, 100);
+
+    const src_dir = b.build_root.handle;
+    for (subdirs) |s| {
+        var dir = try src_dir.openDir(b.fmt("src/nvim/{s}", .{s}), .{ .iterate = true });
+        defer dir.close();
+        var it = dir.iterateAssumeFirstIteration();
+        while (try it.next()) |entry| {
+            if (entry.name.len < 3) continue;
+            if (entry.name[0] < 'a' or entry.name[0] > 'z') continue;
+            if (std.mem.eql(u8, ".c", entry.name[entry.name.len - 2 ..])) {
+                try nvim_sources.append(b.fmt("{s}/{s}", .{ s, entry.name }));
+            }
+        }
+    }
+
+    for (nvim_sources.items) |s| {
+        const gen_step = try generate_header_for(b, s, nlua0, include_path, target, wip_step);
+        wip_step.dependOn(&gen_step.step);
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/nvim/generators/gen_ex_cmds.lua"));
+        gen_header(b, gen_step, "ex_cmds_enum.generated.h", wip_step);
+        gen_header(b, gen_step, "ex_cmds_defs.generated.h", wip_step);
+        gen_step.addFileArg(b.path("src/nvim/ex_cmds.lua"));
+        wip_step.dependOn(&gen_step.step);
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/nvim/generators/gen_options.lua"));
+        gen_header(b, gen_step, "options.generated.h", wip_step);
+        gen_step.addFileArg(b.path("src/nvim/options.lua"));
+        wip_step.dependOn(&gen_step.step);
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/nvim/generators/gen_options_enum.lua"));
+        gen_header(b, gen_step, "options_enum.generated.h", wip_step);
+        gen_header(b, gen_step, "options_map.generated.h", wip_step);
+        gen_step.addFileArg(b.path("src/nvim/options.lua"));
+        wip_step.dependOn(&gen_step.step);
+    }
+
+    {
+        // TODO: LUAC_PRG is missing. tricky with cross-compiling..
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/nvim/generators/gen_char_blob.lua"));
+        gen_step.addArg("-c");
+        gen_header(b, gen_step, "lua/vim_module.generated.h", wip_step);
+        // NB: vim._init_packages and vim.inspect must be be first and second ones
+        // respectively, otherwise --luamod-dev won't work properly.
+        const names = [_][]const u8{
+            "_init_packages",
+            "inspect",
+            "_editor",
+            "filetype",
+            "fs",
+            "F",
+            "keymap",
+            "loader",
+            "_defaults",
+            "_options",
+            "shared",
+        };
+        for (names) |n| {
+            gen_step.addFileArg(b.path(b.fmt("runtime/lua/vim/{s}.lua", .{n})));
+            gen_step.addArg(b.fmt("vim.{s}", .{n}));
+        }
+        wip_step.dependOn(&gen_step.step);
+    }
+}
+
+pub fn gen_header(b: *std.Build, gen_step: *std.Build.Step.Run, name: []const u8, wip_step: *std.Build.Step) void {
+    const header = gen_step.addOutputFileArg(name);
+
+    // TODO: these public installations are not intentional. need to collect these outputs to the next step..
+    wip_step.dependOn(&b.addInstallHeaderFile(header, name).step); // DEBUG
+}
+
+pub const PreprocessorOptions = struct {
+    include_dirs: []const LazyPath = &.{},
+    c_macros: []const []const u8 = &.{},
+    target: ?std.Build.ResolvedTarget = null,
+};
+
+// TODO: this should be suggested to upstream
+pub fn run_preprocessor(
+    b: *std.Build,
+    src: LazyPath,
+    output_name: []const u8,
+    options: PreprocessorOptions,
+) !LazyPath {
+    const run_step = std.Build.Step.Run.create(b, b.fmt("preprocess to get {s}", .{output_name}));
+    run_step.addArgs(&.{ b.graph.zig_exe, "cc", "-E" });
+    run_step.addFileArg(src);
+    run_step.addArg("-o");
+    const output = run_step.addOutputFileArg(output_name);
+    // TODO: include path logic for addCSourceFiles and TranslateC is _very_ different
+    for (options.include_dirs) |include_dir| {
+        run_step.addArg("-I");
+        run_step.addDirectoryArg(include_dir);
+    }
+    for (options.c_macros) |c_macro| {
+        run_step.addArg(b.fmt("-D{s}", .{c_macro}));
+        run_step.addArg(c_macro);
+    }
+    if (options.target) |t| {
+        if (!t.query.isNative()) {
+            run_step.addArgs(&.{
+                "-target", try t.query.zigTriple(b.allocator),
+            });
+        }
+    }
+    run_step.addArgs(&.{ "-MMD", "-MF" });
+    _ = run_step.addDepFileOutputArg(b.fmt("{s}.d", .{output_name}));
+    return output;
+}
+
+pub fn generate_header_for(
+    b: *std.Build,
+    name: []const u8,
+    nlua0: *std.Build.Step.Compile,
+    include_path: []const LazyPath,
+    target: ?std.Build.ResolvedTarget,
+    step: *std.Build.Step,
+) !*std.Build.Step.Run {
+    if (name.len < 2 or !std.mem.eql(u8, ".c", name[name.len - 2 ..])) return error.InvalidBaseName;
+    const basename = name[0 .. name.len - 2];
+    // TODO: need .deps/usr/include and zig_lua include dirs
+    const input_file = b.path(b.fmt("src/nvim/{s}", .{name}));
+    const i_file = try run_preprocessor(b, input_file, b.fmt("{s}.i", .{basename}), .{
+        .include_dirs = include_path,
+        .c_macros = &.{ "HAVE_UNIBILIUM", "_GNU_SOURCE" },
+        .target = target,
+    });
+    const run_step = b.addRunArtifact(nlua0);
+    run_step.addFileArg(b.path("src/nvim/generators/gen_declarations.lua"));
+    run_step.addFileArg(input_file);
+    gen_header(b, run_step, b.fmt("{s}.c.generated.h", .{basename}), step);
+    gen_header(b, run_step, b.fmt("{s}.h.generated.h", .{basename}), step);
+
+    run_step.addFileArg(i_file);
+    return run_step;
+}
+
+pub fn build_nlua0(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    use_luajit: bool,
+    ziglua: *std.Build.Dependency,
+    libluv: *std.Build.Step.Compile,
+    lpeg: *std.Build.Dependency,
+) *std.Build.Step.Compile {
+    const options = b.addOptions();
+    options.addOption(bool, "use_luajit", use_luajit);
+
+    const nlua0_exe = b.addExecutable(.{
+        .name = "nlua0",
+        .root_source_file = .{ .path = "src/nlua0.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+    const nlua0_mod = &nlua0_exe.root_module;
+
+    const exe_unit_tests = b.addTest(.{
+        .root_source_file = .{ .path = "src/nlua0.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+
+    // TODO: need mod.addLibraryPathFromDependency(ziglua)
+    // nlua0_mod.include_dirs.append(nlua0_mod.owner.allocator, .{ .other_step = ziglua_mod }) catch @panic("OOM");
+
+    const flags = [_][]const u8{
+        // Standard version used in Lua Makefile
+        "-std=gnu99",
+    };
+
+    const embedded_data = b.addModule("embedded_data", .{
+        .root_source_file = .{ .path = "runtime/embedded_data.zig" },
+    });
+
+    for ([2]*std.Build.Module{ nlua0_mod, &exe_unit_tests.root_module }) |mod| {
+        mod.addImport("ziglua", ziglua.module("ziglua"));
+        mod.addImport("embedded_data", embedded_data);
+        // addImport already links by itself. but we need headers as well..
+        mod.linkLibrary(ziglua.artifact("lua"));
+
+        mod.linkLibrary(libluv);
+
+        mod.addOptions("options", options);
+
+        mod.addIncludePath(b.path("src"));
+        mod.addIncludePath(b.path("src/includes_fixmelater"));
+        mod.addIncludePath(lpeg.path(""));
+        mod.addCSourceFiles(.{
+            .files = &.{
+                "src/mpack/lmpack.c",
+                "src/mpack/mpack_core.c",
+                "src/mpack/object.c",
+                "src/mpack/conv.c",
+                "src/mpack/rpc.c",
+            },
+            .flags = &flags,
+        });
+        mod.addCSourceFiles(.{
+            .root = .{ .dependency = .{ .dependency = lpeg, .sub_path = "" } },
+            .files = &.{
+                "lpcap.c",
+                "lpcode.c",
+                "lpprint.c",
+                "lptree.c",
+                "lpvm.c",
+            },
+            .flags = &flags,
+        });
+
+        if (!use_luajit) {
+            mod.addCSourceFiles(.{
+                .files = &.{
+                    "src/bit.c",
+                },
+                .flags = &flags,
+            });
+        }
+    }
+
+    // for debugging the nlua0 environment
+    // like this: `zig build nlua0 -- script.lua {args}`
+    const run_cmd = b.addRunArtifact(nlua0_exe);
+    if (b.args) |args| {
+        run_cmd.addArgs(args);
+    }
+    const run_step = b.step("nlua0", "Run nlua0 build tool");
+    run_step.dependOn(&run_cmd.step);
+
+    b.installArtifact(nlua0_exe); // DEBUG
+
+    const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests);
+
+    const test_step = b.step("test_nlua0", "Run unit tests for nlua0");
+    test_step.dependOn(&run_exe_unit_tests.step);
+    return nlua0_exe;
+}
+
+pub fn build_libluv(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    lua: *std.Build.Step.Compile,
+    libuv: *std.Build.Step.Compile,
+) !*std.Build.Step.Compile {
+    const upstream = b.dependency("libluv", .{});
+    const compat53 = b.dependency("lua_compat53", .{});
+    const lib = b.addStaticLibrary(.{
+        .name = "luv",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    lib.linkLibrary(lua);
+    lib.linkLibrary(libuv);
+
+    lib.addIncludePath(upstream.path("src"));
+    lib.addIncludePath(compat53.path("c-api"));
+
+    lib.installHeader(upstream.path("src/luv.h"), "luv/luv.h");
+
+    lib.addCSourceFiles(.{ .root = upstream.path("src/"), .files = &.{
+        "luv.c",
+    } });
+
+    lib.addCSourceFiles(.{ .root = compat53.path("c-api"), .files = &.{
+        "compat-5.3.c",
+    } });
+
+    return lib;
+}

build.zig.zon

@@ -0,0 +1,51 @@
+.{
+    .name = "neovim",
+    .version = "0.0.0",
+
+    // This field is optional.
+    // This is currently advisory only; Zig does not yet do anything
+    // with this value.
+    //.minimum_zig_version = "0.12.0",
+
+    // lpeg fetching from https://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.2.tar.gz doesn't work, bundlevendor for now
+
+    .dependencies = .{
+        .ziglua = .{
+            .url = "https://github.com/natecraddock/ziglua/archive/486f51d3acc61d805783f5f07aee34c75ab59a25.tar.gz",
+            .hash = "12208603e0f51fa6ce7201d8e851c5979b6b78887434623ac87a0f2a5a3dd3e75130",
+        },
+        .lpeg = .{
+            .url = "lpeg-1.0.2.tar.gz",
+            .hash = "1220278000ca8387182d5be708dfa2eec93ada79a058d5f394a27457690e40b318e9",
+        },
+        .libluv = .{
+            .url = "https://github.com/luvit/luv/releases/download/1.48.0-2/luv-1.48.0-2.tar.gz",
+            .hash = "122050b45fc1b2d1e72d30d3e4f446735ab95bbb88658bc1de736e5dc71c3e3cd767",
+        },
+        .lua_compat53 = .{
+            .url = "https://github.com/lunarmodules/lua-compat-5.3/archive/v0.13.tar.gz",
+            .hash = "1220e75685f00c242fbf6c1c60e98dd1b24ba99e38277970a10636e44ed08cf2af8a",
+        },
+        .treesitter = .{
+            .url = "https://github.com/bfredl/tree-sitter/archive/1174f67cc6a9661fdd59f847d690ee874d8eaa66.tar.gz",
+            .hash = "1220cec07c0fd62e3832c13eb1744c4c94f8b9915d86de960ce849f48875fa61dec9",
+        },
+        .libuv = .{ .path = "./deps/libuv" },
+        .msgpack_c = .{ .path = "./deps/msgpack_c/" },
+        .unibilium = .{ .path = "./deps/unibilium/" },
+        .vterm = .{ .path = "./deps/vterm/" },
+    },
+    .paths = .{
+        // This makes *all* files, recursively, included in this package. It is generally
+        // better to explicitly list the files and directories instead, to insure that
+        // fetching from tarballs, file system paths, and version control all result
+        // in the same contents hash.
+        "",
+        // For example...
+        //"build.zig",
+        //"build.zig.zon",
+        //"src",
+        //"LICENSE",
+        //"README.md",
+    },
+}

deps/libuv/build.zig

@@ -0,0 +1,124 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const upstream = b.dependency("libuv", .{});
+    const lib = b.addStaticLibrary(.{
+        .name = "uv",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    lib.addIncludePath(upstream.path("include"));
+    lib.addIncludePath(upstream.path("src"));
+
+    lib.installHeadersDirectory(upstream.path("include"), ".", .{});
+
+    if (target.result.os.tag == .windows) {
+        lib.linkSystemLibrary("psapi");
+        lib.linkSystemLibrary("user32");
+        lib.linkSystemLibrary("advapi32");
+        lib.linkSystemLibrary("iphlpapi");
+        lib.linkSystemLibrary("userenv");
+        lib.linkSystemLibrary("ws2_32");
+    }
+    if (target.result.os.tag == .linux) {
+        lib.linkSystemLibrary("pthread");
+    }
+    lib.linkLibC();
+
+    if (target.result.os.tag != .windows) {
+        lib.root_module.addCMacro("FILE_OFFSET_BITS", "64");
+        lib.root_module.addCMacro("_LARGEFILE_SOURCE", "");
+    }
+
+    if (target.result.os.tag == .linux) {
+        lib.root_module.addCMacro("_GNU_SOURCE", "");
+        lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112");
+    }
+
+    if (target.result.isDarwin()) {
+        lib.root_module.addCMacro("_DARWIN_UNLIMITED_SELECT", "1");
+        lib.root_module.addCMacro("_DARWIN_USE_64_BIT_INODE", "1");
+    }
+
+    const root = upstream.path("");
+
+    // C files common to all platforms
+    lib.addCSourceFiles(.{ .root = root, .files = &.{
+        "src/fs-poll.c",
+        "src/idna.c",
+        "src/inet.c",
+        "src/random.c",
+        "src/strscpy.c",
+        "src/strtok.c",
+        "src/threadpool.c",
+        "src/timer.c",
+        "src/uv-common.c",
+        "src/uv-data-getter-setters.c",
+        "src/version.c",
+    } });
+
+    if (target.result.os.tag != .windows) {
+        lib.addCSourceFiles(.{ .root = root, .files = &.{
+            "src/unix/async.c",
+            "src/unix/core.c",
+            "src/unix/dl.c",
+            "src/unix/fs.c",
+            "src/unix/getaddrinfo.c",
+            "src/unix/getnameinfo.c",
+            "src/unix/loop-watcher.c",
+            "src/unix/loop.c",
+            "src/unix/pipe.c",
+            "src/unix/poll.c",
+            "src/unix/process.c",
+            "src/unix/random-devurandom.c",
+            "src/unix/signal.c",
+            "src/unix/stream.c",
+            "src/unix/tcp.c",
+            "src/unix/thread.c",
+            "src/unix/tty.c",
+            "src/unix/udp.c",
+        } });
+    }
+
+    if (target.result.os.tag == .linux or target.result.isDarwin()) {
+        lib.addCSourceFiles(.{ .root = root, .files = &.{
+            "src/unix/proctitle.c",
+        } });
+    }
+
+    if (target.result.os.tag == .linux) {
+        lib.addCSourceFiles(.{ .root = root, .files = &.{
+            "src/unix/linux.c",
+            "src/unix/procfs-exepath.c",
+            "src/unix/random-getrandom.c",
+            "src/unix/random-sysctl-linux.c",
+        } });
+    }
+
+    if (target.result.isBSD()) {
+        lib.addCSourceFiles(.{ .root = root, .files = &.{
+            "src/unix/bsd-ifaddrs.c",
+            "src/unix/kqueue.c",
+        } });
+    }
+
+    if (target.result.isDarwin() or target.result.os.tag == .openbsd) {
+        lib.addCSourceFiles(.{ .root = root, .files = &.{
+            "src/unix/random-getentropy.c",
+        } });
+    }
+
+    if (target.result.isDarwin()) {
+        lib.addCSourceFiles(.{ .root = root, .files = &.{
+            "src/unix/darwin-proctitle.c",
+            "src/unix/darwin.c",
+            "src/unix/fsevents.c",
+        } });
+    }
+
+    b.installArtifact(lib);
+}

deps/libuv/build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = "libuv",
+    .version = "1.48.0",
+    .paths = .{""},
+
+    .dependencies = .{
+        .libuv = .{
+            .url = "https://github.com/libuv/libuv/archive/v1.48.0.tar.gz",
+            .hash = "1220c2e1269cfee3735a7331f1fa4c13aa3e4f355fca5b402be3ea72a464c76f1324",
+        },
+    },
+}

deps/msgpack_c/build.zig

@@ -0,0 +1,45 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const upstream = b.dependency("msgpack_c", .{});
+    const lib = b.addStaticLibrary(.{
+        .name = "msgpack_c",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    lib.addIncludePath(upstream.path("include"));
+    lib.addIncludePath(upstream.path("src"));
+
+    // TODO: actually detect BIG-lyness of `target`
+    const sysdep = b.addConfigHeader(.{
+        .style = .{ .cmake = upstream.path("cmake/sysdep.h.in") },
+        .include_path = "msgpack/sysdep.h",
+    }, .{ .MSGPACK_ENDIAN_BIG_BYTE = "0", .MSGPACK_ENDIAN_LITTLE_BYTE = "1" });
+    lib.addConfigHeader(sysdep);
+
+    const pack_template = b.addConfigHeader(.{
+        .style = .{ .cmake = upstream.path("cmake/pack_template.h.in") },
+        .include_path = "msgpack/pack_template.h",
+    }, .{ .MSGPACK_ENDIAN_BIG_BYTE = "0", .MSGPACK_ENDIAN_LITTLE_BYTE = "1" });
+    lib.addConfigHeader(pack_template);
+
+    lib.installHeadersDirectory(upstream.path("include"), ".", .{});
+    lib.installConfigHeader(sysdep);
+    lib.installConfigHeader(pack_template);
+
+    lib.linkLibC();
+
+    lib.addCSourceFiles(.{ .root = upstream.path(""), .files = &.{
+        "src/objectc.c",
+        "src/unpack.c",
+        "src/version.c",
+        "src/vrefbuffer.c",
+        "src/zone.c",
+    } });
+
+    b.installArtifact(lib);
+}

deps/msgpack_c/build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = "msgpack_c",
+    .version = "6.0.1",
+    .paths = .{""},
+
+    .dependencies = .{
+        .msgpack_c = .{
+            .url = "https://github.com/msgpack/msgpack-c/archive/c-6.0.1.tar.gz",
+            .hash = "1220781494e79f4d4351608b6442f9f7814df38dea9e96858dadbb54a65e564134c3",
+        },
+    },
+}

deps/unibilium/build.zig

@@ -0,0 +1,27 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const upstream = b.dependency("unibilium", .{});
+    const lib = b.addStaticLibrary(.{
+        .name = "unibilium",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    lib.addIncludePath(upstream.path(""));
+
+    lib.installHeader(upstream.path("unibilium.h"), "unibilium.h");
+
+    lib.linkLibC();
+
+    lib.addCSourceFiles(.{ .root = upstream.path(""), .files = &.{
+        "unibilium.c",
+        "uninames.c",
+        "uniutil.c",
+    } });
+
+    b.installArtifact(lib);
+}

deps/unibilium/build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = "unibilium",
+    .version = "0.0.1",
+    .paths = .{""},
+
+    .dependencies = .{
+        .unibilium = .{
+            .url = "https://github.com/neovim/unibilium/archive/d72c3598e7ac5d1ebf86ee268b8b4ed95c0fa628.tar.gz",
+            .hash = "1220eb029114911f764a2266a8595829343b726d8aa81105a728920fdcba4b482a7d",
+        },
+    },
+}

deps/vterm/build.zig

@@ -0,0 +1,36 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const upstream = b.dependency("vterm", .{});
+    const lib = b.addStaticLibrary(.{
+        .name = "vterm",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    lib.addIncludePath(upstream.path("src/"));
+    lib.addIncludePath(upstream.path("include/"));
+    // local additions: DECdrawing.inc and uk.inc (u w8t m8)
+    lib.addIncludePath(b.path("include/"));
+
+    lib.installHeadersDirectory(upstream.path("include"), ".", .{});
+
+    lib.linkLibC();
+
+    lib.addCSourceFiles(.{ .root = upstream.path("src"), .files = &.{
+        "encoding.c",
+        "keyboard.c",
+        "mouse.c",
+        "parser.c",
+        "pen.c",
+        "screen.c",
+        "state.c",
+        "unicode.c",
+        "vterm.c",
+    } });
+
+    b.installArtifact(lib);
+}

deps/vterm/build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = "vterm",
+    .version = "0.3.3",
+    .paths = .{""},
+
+    .dependencies = .{
+        .vterm = .{
+            .url = "https://github.com/neovim/libvterm/archive/v0.3.3.tar.gz",
+            .hash = "1220184ae54d2331deb1be5d905978205c2731714e4b0ae8587e72c4ce02da317195",
+        },
+    },
+}

deps/vterm/include/encoding/DECdrawing.inc

@@ -0,0 +1,36 @@
+ static const struct StaticTableEncoding encoding_DECdrawing = {
+   { .decode = &decode_table },
+   {
+     [0x60] = 0x25C6,
+     [0x61] = 0x2592,
+     [0x62] = 0x2409,
+     [0x63] = 0x240C,
+     [0x64] = 0x240D,
+     [0x65] = 0x240A,
+     [0x66] = 0x00B0,
+     [0x67] = 0x00B1,
+     [0x68] = 0x2424,
+     [0x69] = 0x240B,
+     [0x6a] = 0x2518,
+     [0x6b] = 0x2510,
+     [0x6c] = 0x250C,
+     [0x6d] = 0x2514,
+     [0x6e] = 0x253C,
+     [0x6f] = 0x23BA,
+     [0x70] = 0x23BB,
+     [0x71] = 0x2500,
+     [0x72] = 0x23BC,
+     [0x73] = 0x23BD,
+     [0x74] = 0x251C,
+     [0x75] = 0x2524,
+     [0x76] = 0x2534,
+     [0x77] = 0x252C,
+     [0x78] = 0x2502,
+     [0x79] = 0x2A7D,
+     [0x7a] = 0x2A7E,
+     [0x7b] = 0x03C0,
+     [0x7c] = 0x2260,
+     [0x7d] = 0x00A3,
+     [0x7e] = 0x00B7,
+   }
+ };

deps/vterm/include/encoding/uk.inc

@@ -0,0 +1,6 @@
+ static const struct StaticTableEncoding encoding_uk = {
+   { .decode = &decode_table },
+   {
+     [0x23] = 0x00a3,
+   }
+ };

runtime/doc/builtin.txt

@@ -8208,6 +8208,10 @@ synconcealed({lnum}, {col})                                     *synconcealed()*
 			synconcealed(lnum, 5)   [1, 'X', 2]
 			synconcealed(lnum, 6)   [0, '', 0]
 
+		Note: Doesn't consider |matchadd()| highlighting items,
+		since syntax and matching highlighting are two different
+		mechanisms |syntax-vs-match|.
+
 synstack({lnum}, {col})                                             *synstack()*
 		Return a |List|, which is the stack of syntax items at the
 		position {lnum} and {col} in the current window.  {lnum} is

runtime/doc/pattern.txt

@@ -1375,6 +1375,19 @@ Finally, these constructs are unique to Perl:
 ==============================================================================
 10. Highlighting matches				*match-highlight*
 
+							*syntax-vs-match*
+		Note that the match highlight mechanism is independent
+		of |syntax-highlighting|, which is (usually) a buffer-local
+		highlighting, while matching is window-local, both methods
+		can be freely mixed.  Match highlighting functions give you
+		a bit more flexibility in when and how to apply, but are
+		typically only used for temporary highlighting, without strict
+		rules.  Both methods can be used to conceal text.
+
+		Thus the matching functions like |matchadd()| won't consider
+		syntax rules and functions like |synconcealed()| and the
+		other way around.
+
 							*:mat* *:match*
 :mat[ch] {group} /{pattern}/
 		Define a pattern to highlight in the current window.  It will

runtime/doc/syntax.txt

@@ -3825,7 +3825,9 @@ Whether or not it is actually concealed depends on the value of the
 'conceallevel' option.  The 'concealcursor' option is used to decide whether
 concealable items in the current line are displayed unconcealed to be able to
 edit the line.
-Another way to conceal text is with |matchadd()|.
+
+Another way to conceal text is with |matchadd()|, but internally this works a
+bit differently |syntax-vs-match|.
 
 concealends						*:syn-concealends*
 
@@ -3833,7 +3835,9 @@ When the "concealends" argument is given, the start and end matches of
 the region, but not the contents of the region, are marked as concealable.
 Whether or not they are actually concealed depends on the setting on the
 'conceallevel' option. The ends of a region can only be concealed separately
-in this way when they have their own highlighting via "matchgroup"
+in this way when they have their own highlighting via "matchgroup".  The
+|synconcealed()| function can be used to retrieve information about conealed
+items.
 
 cchar							*:syn-cchar*
 							*E844*

runtime/doc/usr_41.txt

@@ -948,7 +948,7 @@ Syntax and highlighting:	  *syntax-functions* *highlighting-functions*
 	synIDattr()		get a specific attribute of a syntax ID
 	synIDtrans()		get translated syntax ID
 	synstack()		get list of syntax IDs at a specific position
-	synconcealed()		get info about concealing
+	synconcealed()		get info about (syntax) concealing
 	diff_hlID()		get highlight ID for diff mode at a position
 	matchadd()		define a pattern to highlight (a "match")
 	matchaddpos()		define a list of positions to highlight

runtime/embedded_data.zig

@@ -0,0 +1,2 @@
+pub const inspect_module = @embedFile("./lua/vim/inspect.lua");
+pub const shared_module = @embedFile("./lua/vim/shared.lua");

runtime/lua/vim/_meta/vimfn.lua

@@ -9752,6 +9752,10 @@ function vim.fn.synIDtrans(synID) end
 ---   synconcealed(lnum, 5)   [1, 'X', 2]
 ---   synconcealed(lnum, 6)   [0, '', 0]
 ---
+--- Note: Doesn't consider |matchadd()| highlighting items,
+--- since syntax and matching highlighting are two different
+--- mechanisms |syntax-vs-match|.
+---
 --- @param lnum integer
 --- @param col integer
 --- @return {[1]: integer, [2]: string, [3]: integer}

src/nlua0.zig

@@ -0,0 +1,117 @@
+const std = @import("std");
+const ziglua = @import("ziglua");
+const options = @import("options");
+
+const embedded_data = @import("embedded_data");
+
+const hashy = @embedFile("nvim/generators/hashy.lua");
+
+const Lua = ziglua.Lua;
+
+extern "c" fn luaopen_mpack(ptr: *anyopaque) c_int;
+extern "c" fn luaopen_lpeg(ptr: *anyopaque) c_int;
+extern "c" fn luaopen_luv(ptr: *anyopaque) c_int;
+extern "c" fn luaopen_bit(ptr: *anyopaque) c_int;
+
+fn init() !*Lua {
+    // Initialize the Lua vm
+    var lua = try Lua.newStateLibc();
+    lua.openLibs();
+
+    // this sets _G.vim by itself, so we don't need to
+    try lua.loadBuffer(embedded_data.shared_module, "shared.lua");
+    lua.call(0, 1);
+
+    try lua.loadBuffer(embedded_data.inspect_module, "inspect.lua");
+    lua.call(0, 1);
+    lua.setField(-2, "inspect");
+
+    _ = try lua.getGlobal("package");
+    _ = lua.getField(-1, "preload");
+    try lua.loadBuffer(hashy, "hashy.lua"); // [package, preload, hashy]
+    lua.setField(-2, "generators.hashy");
+    lua.pop(2);
+
+    const retval = luaopen_mpack(lua);
+    if (retval != 1) return error.LoadError;
+    lua.setField(-2, "mpack");
+
+    // NB: not fully supportg luv threads and things like that, but this is just to demo uv+luv
+    const retval_luv = luaopen_luv(lua);
+    if (retval_luv != 1) return error.LoadError;
+    lua.setField(-2, "uv");
+
+    const retval2 = luaopen_lpeg(lua);
+    if (retval2 != 1) return error.LoadError;
+    lua.setField(-3, "lpeg");
+
+    lua.pop(2);
+
+    if (!options.use_luajit) {
+        lua.pop(luaopen_bit(lua));
+    }
+    return lua;
+}
+
+pub fn main() !void {
+    const argv = std.os.argv;
+
+    const lua = try init();
+    defer lua.deinit();
+
+    if (argv.len < 2) {
+        std.debug.print("USAGE: nlua0 script.lua args...\n\n", .{});
+        try lua.doString("print(vim.inspect(vim.uv.os_uname()))");
+        return;
+    }
+    lua.createTable(@intCast(argv.len - 2), 1);
+    for (0.., argv[1..]) |i, arg| {
+        _ = lua.pushString(std.mem.span(arg));
+        lua.rawSetIndex(-2, @intCast(i));
+    }
+    lua.setGlobal("arg");
+
+    // Create an allocator
+    // var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    // const allocator = gpa.allocator();
+    // defer _ = gpa.deinit();
+
+    //std.debug.print("All your {s} are belong to us.\n", .{argv[1]});
+
+    _ = try lua.getGlobal("debug");
+    _ = lua.getField(-1, "traceback");
+    try lua.loadFile(std.mem.span(argv[1]));
+    lua.protectedCall(0, 0, -2) catch |e| {
+        if (e == error.Runtime) {
+            const msg = try lua.toString(-1);
+            std.debug.print("{s}\n", .{msg});
+        }
+        return e;
+    };
+}
+
+fn do_ret1(lua: *Lua, str: [:0]const u8) !void {
+    try lua.loadString(str);
+    try lua.protectedCall(0, 1, 0);
+}
+
+test "simple test" {
+    const lua = try init();
+    defer lua.deinit();
+
+    try do_ret1(lua, "return vim.isarray({2,3})");
+    try std.testing.expectEqual(true, lua.toBoolean(-1));
+    lua.pop(1);
+
+    try do_ret1(lua, "return vim.isarray({a=2,b=3})");
+    try std.testing.expectEqual(false, lua.toBoolean(-1));
+    lua.pop(1);
+
+    try do_ret1(lua, "return vim.inspect(vim.mpack.decode('\\146\\42\\69'))");
+    try std.testing.expectEqualStrings("{ 42, 69 }", try lua.toString(-1));
+    lua.pop(1);
+
+    try do_ret1(lua, "return require'bit'.band(7,12)");
+    try std.testing.expectEqualStrings("4", try lua.toString(-1));
+    lua.pop(1);
+}

src/nvim/CMakeLists.txt

@@ -664,7 +664,7 @@ list(APPEND NVIM_GENERATED_SOURCES
 )
 
 add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS}
-  COMMAND ${LUA_GEN} ${EX_CMDS_GENERATOR} ${GENERATED_INCLUDES_DIR} ${GENERATED_DIR}
+  COMMAND ${LUA_GEN} ${EX_CMDS_GENERATOR} ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS} ${CMAKE_CURRENT_LIST_DIR}/ex_cmds.lua
   DEPENDS ${LUA_GEN_DEPS} ${EX_CMDS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/ex_cmds.lua
 )
 
@@ -681,12 +681,12 @@ add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
 )
 
 add_custom_command(OUTPUT ${GENERATED_OPTIONS}
-  COMMAND ${LUA_GEN} ${OPTIONS_GENERATOR} ${GENERATED_OPTIONS}
+  COMMAND ${LUA_GEN} ${OPTIONS_GENERATOR} ${GENERATED_OPTIONS} ${CMAKE_CURRENT_LIST_DIR}/options.lua
   DEPENDS ${LUA_GEN_DEPS} ${OPTIONS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/options.lua
 )
 
 add_custom_command(OUTPUT ${GENERATED_OPTIONS_ENUM} ${GENERATED_OPTIONS_MAP}
-  COMMAND ${LUA_GEN} ${OPTIONS_ENUM_GENERATOR} ${GENERATED_OPTIONS_ENUM} ${GENERATED_OPTIONS_MAP}
+  COMMAND ${LUA_GEN} ${OPTIONS_ENUM_GENERATOR} ${GENERATED_OPTIONS_ENUM} ${GENERATED_OPTIONS_MAP} ${CMAKE_CURRENT_LIST_DIR}/options.lua
   DEPENDS ${LUA_GEN_DEPS} ${OPTIONS_ENUM_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/options.lua
 )
 

src/nvim/buffer_defs.h

@@ -47,8 +47,8 @@ typedef struct {
 #define VALID_VIRTCOL   0x04    // w_virtcol (file col) is valid
 #define VALID_CHEIGHT   0x08    // w_cline_height and w_cline_folded valid
 #define VALID_CROW      0x10    // w_cline_row is valid
-#define VALID_BOTLINE   0x20    // w_botine and w_empty_rows are valid
-#define VALID_BOTLINE_AP 0x40   // w_botine is approximated
+#define VALID_BOTLINE   0x20    // w_botline and w_empty_rows are valid
+#define VALID_BOTLINE_AP 0x40   // w_botline is approximated
 #define VALID_TOPLINE   0x80    // w_topline is valid (for cursor position)
 
 // flags for b_flags

src/nvim/eval.lua

@@ -11621,6 +11621,10 @@ M.funcs = {
       	synconcealed(lnum, 4)   [1, 'X', 2]
       	synconcealed(lnum, 5)   [1, 'X', 2]
       	synconcealed(lnum, 6)   [0, '', 0]
+
+      Note: Doesn't consider |matchadd()| highlighting items,
+      since syntax and matching highlighting are two different
+      mechanisms |syntax-vs-match|.
     ]=],
     name = 'synconcealed',
     params = { { 'lnum', 'integer' }, { 'col', 'integer' } },

src/nvim/generators/gen_ex_cmds.lua

@@ -1,17 +1,15 @@
-local includedir = arg[1]
-local autodir = arg[2]
-
 -- Will generate files ex_cmds_enum.generated.h with cmdidx_T enum
 -- and ex_cmds_defs.generated.h with main Ex commands definitions.
 
-local enumfname = includedir .. '/ex_cmds_enum.generated.h'
-local defsfname = autodir .. '/ex_cmds_defs.generated.h'
+local enumfname = arg[1] -- '/ex_cmds_enum.generated.h'
+local defsfname = arg[2] -- '/ex_cmds_defs.generated.h'
+local ex_cmds_name = arg[3] -- 'ex_cmds.lua'
 
 local enumfile = io.open(enumfname, 'w')
 local defsfile = io.open(defsfname, 'w')
 
 local bit = require 'bit'
-local ex_cmds = require('ex_cmds')
+local ex_cmds = loadfile(ex_cmds_name)()
 local defs = ex_cmds.cmds
 local flags = ex_cmds.flags
 

src/nvim/generators/gen_options.lua

@@ -1,4 +1,5 @@
 local options_file = arg[1]
+local options_input_file = arg[2]
 
 local opt_fd = assert(io.open(options_file, 'w'))
 
@@ -11,7 +12,7 @@ local w = function(s)
 end
 
 --- @module 'nvim.options'
-local options = require('options')
+local options = loadfile(options_input_file)()
 
 local cstr = options.cstr
 

src/nvim/generators/gen_options_enum.lua

@@ -4,6 +4,8 @@
 
 local options_enum_file = arg[1]
 local options_map_file = arg[2]
+local options_input_file = arg[3]
+
 local options_enum_fd = assert(io.open(options_enum_file, 'w'))
 local options_map_fd = assert(io.open(options_map_file, 'w'))
 
@@ -27,7 +29,7 @@ local lowercase_to_titlecase = function(s)
 end
 
 --- @type vim.option_meta[]
-local options = require('options').options
+local options = loadfile(options_input_file)().options
 
 -- Generate BV_ enum constants.
 enum_w('/// "indir" values for buffer-local options.')

src/nvim/window.c

@@ -2467,6 +2467,7 @@ void win_init_empty(win_T *wp)
   wp->w_topline = 1;
   wp->w_topfill = 0;
   wp->w_botline = 2;
+  wp->w_valid = 0;
   wp->w_s = &wp->w_buffer->b_s;
 }
 

test/old/testdir/test_autocmd.vim

@@ -4105,4 +4105,16 @@ func Test_SwapExists_set_other_buf_modified()
   bwipe!
 endfunc
 
+func Test_BufEnter_botline()
+  set hidden
+  call writefile(range(10), 'Xxx1', 'D')
+  call writefile(range(20), 'Xxx2', 'D')
+  edit Xxx1
+  edit Xxx2
+  au BufEnter Xxx1 call assert_true(line('w$') > 1)
+  edit Xxx1
+  au! BufEnter Xxx1
+  set hidden&vim
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab