From 3d33735d73de269278fb87d2c9ae8fb7d83977be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Feb 2026 00:29:31 -0800 Subject: [PATCH] zig build: add --fork CLI argument closes #31124 --- lib/compiler/build_runner.zig | 49 ++++++++------- src/Package.zig | 37 +++++++++++ src/Package/Fetch.zig | 45 +++++--------- src/Package/Manifest.zig | 46 +++++++++++++- src/main.zig | 113 +++++++++++++++++++++++++++++----- 5 files changed, 221 insertions(+), 69 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 2dd18d4f0d..4d6b640e88 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -1573,6 +1573,23 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ -fsys=[name] Enable a system integration \\ -fno-sys=[name] Disable a system integration \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc + \\ (e.g. glibc or musl) built for multiple foreign + \\ architectures, allowing execution of non-native + \\ programs that link with libc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ \\ Available System Integrations: Enabled: \\ ); @@ -1592,33 +1609,16 @@ fn printUsage(b: *std.Build, w: *Writer) !void { try w.writeAll( \\ \\General Options: + \\ -h, --help Print this help and exit + \\ -l, --list-steps Print available steps + \\ \\ -p, --prefix [path] Where to install files (default: zig-out) \\ --prefix-lib-dir [path] Where to install libraries \\ --prefix-exe-dir [path] Where to install executables \\ --prefix-include-dir [path] Where to install C header files - \\ \\ --release[=mode] Request release mode, optionally specifying a \\ preferred optimization mode: fast, safe, small \\ - \\ -fdarling, -fno-darling Integration with system-installed Darling to - \\ execute macOS programs on Linux hosts - \\ (default: no) - \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute - \\ foreign-architecture programs on Linux hosts - \\ (default: no) - \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc - \\ (e.g. glibc or musl) built for multiple foreign - \\ architectures, allowing execution of non-native - \\ programs that link with libc. - \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on - \\ ARM64 macOS hosts. (default: no) - \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to - \\ execute WASI binaries. (default: no) - \\ -fwine, -fno-wine Integration with system-installed Wine to execute - \\ Windows programs on Linux hosts. (default: no) - \\ - \\ -h, --help Print this help and exit - \\ -l, --list-steps Print available steps \\ --verbose Print commands before executing them \\ --color [auto|off|on] Enable or disable colored error messages \\ --error-style [style] Control how build errors are printed @@ -1641,9 +1641,6 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. \\ The timeout must include a unit: ns, us, ms, s, m, h - \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit - \\ needed (Default) Lazy dependencies are fetched as needed - \\ all Lazy dependencies are always fetched \\ --watch Continuously rebuild when source files are modified \\ --debounce Delay before rebuilding after changed file detected \\ --webui[=ip] Enable the web interface on the given IP address @@ -1656,6 +1653,12 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ -fincremental Enable incremental compilation \\ -fno-incremental Disable incremental compilation \\ + \\Package Management Options: + \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit + \\ needed (Default) Lazy dependencies are fetched as needed + \\ all Lazy dependencies are always fetched + \\ --fork=[path] Override one or more packages from dependency tree + \\ \\Advanced Options: \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error \\ -fno-reference-trace Disable reference trace diff --git a/src/Package.zig b/src/Package.zig index fda4c1c178..1307b8e9f9 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -137,6 +137,43 @@ pub const Hash = struct { _ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable; return result; } + + pub fn projectId(hash: *const Hash) ProjectId { + const name = std.mem.sliceTo(&hash.bytes, '-'); + const hashplus = hash.bytes[std.mem.findScalarLast(u8, &hash.bytes, '-').? + 1 ..]; + var decoded: [6]u8 = undefined; + std.base64.url_safe_no_pad.Decoder.decode(&decoded, hashplus[0..8]) catch unreachable; + const fingerprint_id = std.mem.readInt(u32, decoded[0..4], .little); + return .init(name, fingerprint_id); + } + + test projectId { + const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw"); + const project_id = hash.projectId(); + + var expected_name: [32]u8 = @splat(0); + expected_name[0.."pulseaudio".len].* = "pulseaudio".*; + try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); + + try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id); + } +}; + +/// Minimum information required to identify whether a package is an artifact +/// of a given project. +pub const ProjectId = struct { + /// Bytes after name.len are set to zero. + padded_name: [32]u8, + fingerprint_id: u32, + + pub fn init(name: []const u8, fingerprint_id: u32) ProjectId { + var padded_name: [32]u8 = @splat(0); + @memcpy(padded_name[0..name.len], name); + return .{ + .padded_name = padded_name, + .fingerprint_id = fingerprint_id, + }; + } }; pub const MultihashFunction = enum(u16) { diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index d873fc9bd9..7aa3cee001 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -142,6 +142,8 @@ pub const JobQueue = struct { /// Set of hashes that will be additionally fetched even if they are marked /// as lazy. unlazy_set: UnlazySet = .{}, + /// Identifies paths that override all packages in the tree matching + fork_set: ForkSet = .{}, pub const Mode = enum { /// Non-lazy dependencies are always fetched. @@ -152,6 +154,7 @@ pub const JobQueue = struct { }; pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch); pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void); + pub const ForkSet = std.AutoArrayHashMapUnmanaged(Package.ProjectId, Cache.Path); pub fn deinit(jq: *JobQueue) void { const io = jq.io; @@ -801,45 +804,29 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { const io = f.job_queue.io; const eb = &f.error_bundle; const arena = f.arena.allocator(); - const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( + const manifest_path = try pkg_root.join(arena, Manifest.basename); + + f.manifest = @as(Manifest, undefined); + + Manifest.load( io, - try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), arena, - .limited(Manifest.max_bytes), - .@"1", - 0, + manifest_path, + &f.manifest_ast, + eb, + &f.manifest.?, + f.allow_missing_paths_field, ) catch |err| switch (err) { error.FileNotFound => return, + error.Canceled => |e| return e, + error.ErrorsBundled => return error.FetchFailed, else => |e| { - const file_path = try pkg_root.join(arena, Manifest.basename); try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }), + .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }), }); return error.FetchFailed; }, }; - - const ast = &f.manifest_ast; - ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); - - if (ast.errors.len > 0) { - const file_path = try std.fmt.allocPrint(arena, "{f}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root}); - try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); - return error.FetchFailed; - } - - const rng: std.Random.IoSource = .{ .io = io }; - - f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{ - .allow_missing_paths_field = f.allow_missing_paths_field, - }); - const manifest = &f.manifest.?; - - if (manifest.errors.len > 0) { - const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename }); - try manifest.copyErrorsIntoBundle(ast.*, src_path, eb); - return error.FetchFailed; - } } fn queueJobsForDeps(f: *Fetch) RunError!void { diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index e66a78ae14..4a71d15c81 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -1,10 +1,13 @@ const Manifest = @This(); + const std = @import("std"); +const Io = std.Io; const mem = std.mem; const Allocator = std.mem.Allocator; const assert = std.debug.assert; const Ast = std.zig.Ast; const testing = std.testing; + const Package = @import("../Package.zig"); pub const max_bytes = 10 * 1024 * 1024; @@ -53,7 +56,7 @@ pub const ParseOptions = struct { pub const Error = Allocator.Error; -pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) Error!Manifest { +pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest { const main_node_index = ast.nodeData(.root).node; var arena_instance = std.heap.ArenaAllocator.init(gpa); @@ -61,7 +64,7 @@ pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) E var p: Parse = .{ .gpa = gpa, - .ast = ast, + .ast = ast.*, .arena = arena_instance.allocator(), .errors = .{}, @@ -578,6 +581,45 @@ const Parse = struct { } }; +pub fn load( + io: Io, + arena: Allocator, + manifest_path: std.Build.Cache.Path, + ast: *std.zig.Ast, + error_bundle: *std.zig.ErrorBundle.Wip, + manifest: *Manifest, + allow_missing_paths_field: bool, +) !void { + const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions( + io, + manifest_path.sub_path, + arena, + .limited(max_bytes), + .@"1", + 0, + ); + + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); + + if (ast.errors.len > 0) { + const file_path = try manifest_path.joinString(arena, ""); + try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle); + return error.ErrorsBundled; + } + + const rng: std.Random.IoSource = .{ .io = io }; + + manifest.* = try parse(arena, ast, rng.interface(), .{ + .allow_missing_paths_field = allow_missing_paths_field, + }); + + if (manifest.errors.len > 0) { + const src_path = try error_bundle.printString("{f}", .{manifest_path}); + try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle); + return error.ErrorsBundled; + } +} + test "basic" { const gpa = testing.allocator; diff --git a/src/main.zig b/src/main.zig index dba52807f5..a59c289ce3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4896,7 +4896,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); - var child_argv = std.array_list.Managed([]const u8).init(arena); + var child_argv: std.ArrayList([]const u8) = .empty; + var forks: std.ArrayList(Fork) = .empty; var reference_trace: ?u32 = null; var debug_compile_errors = false; var verbose_link = (native_os != .wasi or builtin.link_libc) and @@ -4917,24 +4918,24 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, var debug_libc_paths_file: ?[]const u8 = null; const argv_index_exe = child_argv.items.len; - _ = try child_argv.addOne(); + _ = try child_argv.addOne(arena); const self_exe_path = try process.executablePathAlloc(io, arena); - try child_argv.append(self_exe_path); + try child_argv.append(arena, self_exe_path); const argv_index_zig_lib_dir = child_argv.items.len; - _ = try child_argv.addOne(); + _ = try child_argv.addOne(arena); const argv_index_build_file = child_argv.items.len; - _ = try child_argv.addOne(); + _ = try child_argv.addOne(arena); const argv_index_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(); + _ = try child_argv.addOne(arena); const argv_index_global_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(); + _ = try child_argv.addOne(arena); - try child_argv.appendSlice(&.{ + try child_argv.appendSlice(arena, &.{ "--seed", try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}), }); @@ -4955,7 +4956,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, // read this file in the parent to obtain the results, in the case the child // exits with code 3. const results_tmp_file_nonce = std.fmt.hex(randInt(io, u64)); - try child_argv.append("-Z" ++ results_tmp_file_nonce); + try child_argv.append(arena, "-Z" ++ results_tmp_file_nonce); var color: Color = .auto; var n_jobs: ?u32 = null; @@ -5000,11 +5001,19 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, fatal("expected [needed|all] after '--fetch=', found '{s}'", .{ sub_arg, }); + } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| { + try forks.append(arena, .{ + .project_id = undefined, + .path = .{ + .root_dir = .cwd(), + .sub_path = sub_arg, + }, + }); } else if (mem.eql(u8, arg, "--system")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; system_pkg_dir_path = args[i]; - try child_argv.append("--system"); + try child_argv.append(arena, "--system"); continue; } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { @@ -5014,7 +5023,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, reference_trace = null; } else if (mem.eql(u8, arg, "--debug-log")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - try child_argv.appendSlice(args[i .. i + 2]); + try child_argv.appendSlice(arena, args[i .. i + 2]); i += 1; if (!build_options.enable_logging) { warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{}); @@ -5070,7 +5079,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, color = std.meta.stringToEnum(Color, args[i]) orelse { fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] }); }; - try child_argv.appendSlice(&.{ arg, args[i] }); + try child_argv.appendSlice(arena, &.{ arg, args[i] }); continue; } else if (mem.cutPrefix(u8, arg, "-j")) |str| { const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| { @@ -5090,11 +5099,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } else if (mem.eql(u8, arg, "--")) { // The rest of the args are supposed to get passed onto // build runner's `build.args` - try child_argv.appendSlice(args[i..]); + try child_argv.appendSlice(arena, args[i..]); break; } } - try child_argv.append(arg); + try child_argv.append(arena, arg); } } @@ -5182,6 +5191,25 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, defer http_client.deinit(); var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; + var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; + + { + // Populate fork_set. + var group: Io.Group = .init; + defer group.cancel(io); + + for (forks.items) |*fork| + group.async(io, loadFork, .{ io, gpa, fork, color }); + + try group.await(io); + + for (forks.items) |*fork| { + const project_id = fork.project_id catch |err| switch (err) { + error.AlreadyReported => process.exit(1), + }; + try fork_set.put(arena, project_id, fork.path); + } + } // This loop is re-evaluated when the build script exits with an indication that it // could not continue due to missing lazy dependencies. @@ -5518,6 +5546,61 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } } +const Fork = struct { + path: Path, + project_id: error{AlreadyReported}!Package.ProjectId, +}; + +fn loadFork(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { + fork.project_id = loadForkFallible(io, gpa, fork, color) catch |err| switch (err) { + error.Canceled => |e| return e, + error.AlreadyReported => |e| e, + else => |e| e: { + std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); + break :e error.AlreadyReported; + }, + }; +} + +fn loadForkFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !Package.ProjectId { + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + var error_bundle: std.zig.ErrorBundle.Wip = undefined; + try error_bundle.init(gpa); + defer error_bundle.deinit(); + + const manifest_path = try fork.path.join(arena, Package.Manifest.basename); + + var manifest_ast: std.zig.Ast = undefined; + var manifest: Package.Manifest = undefined; + + Package.Manifest.load( + io, + arena, + manifest_path, + &manifest_ast, + &error_bundle, + &manifest, + true, + ) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ErrorsBundled => { + assert(error_bundle.root_list.items.len > 0); + var errors = try error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + return error.AlreadyReported; + }, + else => |e| { + std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); + return error.AlreadyReported; + }, + }; + + return .init(manifest.name, manifest.id); +} + const JitCmdOptions = struct { cmd_name: []const u8, root_src_path: []const u8, @@ -7410,7 +7493,7 @@ fn loadManifest( process.exit(2); } - var manifest = try Package.Manifest.parse(gpa, ast, rng.interface(), .{}); + var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); errdefer manifest.deinit(gpa); if (manifest.errors.len > 0) {