From 27487431224eb14c866cf708ce464fe06a87e975 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 24 Feb 2026 19:15:30 -0800 Subject: [PATCH] Configuration: implement FlagLengthPrefixedList --- lib/compiler/Maker/ScannedConfig.zig | 21 ++- lib/compiler/configure_runner.zig | 52 ++++++- lib/std/zig/Configuration.zig | 203 +++++++++++++++++++-------- 3 files changed, 209 insertions(+), 67 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 489f21917c..18dcf7d00c 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -55,17 +55,24 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(field_value.slice(c), .{}); }, Configuration.Deps => { - var deps_field = try s.beginTuple(.{}); - for (field_value.slice(c)) |dep| { - try deps_field.field(@intFromEnum(dep), .{}); - } - try deps_field.end(); + try printValue(sc, s, []Configuration.Step.Index, field_value.slice(c)); }, Configuration.MaxRss => { try s.value(field_value.toBytes(), .{}); }, else => switch (@typeInfo(Field)) { .int => try s.int(field_value), + .pointer => |info| switch (info.size) { + .slice => { + var slice_field = try s.beginTuple(.{}); + for (field_value) |elem| { + try slice_field.fieldPrefix(); + try printValue(sc, s, info.child, elem); + } + try slice_field.end(); + }, + else => comptime unreachable, + }, .@"enum" => { if (@hasDecl(Field, "storage")) switch (Field.storage) { .extended => { @@ -74,6 +81,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try sub_struct.end(); }, .flag_optional => comptime unreachable, + .flag_length_prefixed_list => comptime unreachable, .enum_optional => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { try s.ident(name); @@ -93,6 +101,9 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(null, .{}); } }, + .flag_length_prefixed_list => { + try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice); + }, .extended => @compileError("TODO"), }, else => @compileError("not implemented: " ++ @typeName(Field)), diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 9bc76da59a..014cfd9539 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -279,6 +279,10 @@ const Serialize = struct { return (try addOptionalLazyPathEnum(s, lp)).unwrap(); } + fn addLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.LazyPath { + return @enumFromInt(@intFromEnum(try addOptionalLazyPathEnum(s, lp))); + } + fn addOptionalSemVer(s: *Serialize, sem_ver: ?std.SemanticVersion) !?Configuration.String { return if (sem_ver) |sv| try s.wc.addSemVer(sv) else null; } @@ -286,6 +290,20 @@ const Serialize = struct { fn addOptionalString(s: *Serialize, opt_slice: ?[]const u8) !?Configuration.String { return if (opt_slice) |slice| try s.wc.addString(slice) else null; } + + fn initStringList(s: *Serialize, list: []const []const u8) ![]const Configuration.String { + const wc = s.wc; + const result = try s.arena.alloc(Configuration.String, list.len); + for (result, list) |*dest, src| dest.* = try wc.addString(src); + return result; + } + + fn initOptionalStringList(s: *Serialize, list: []const ?[]const u8) ![]const Configuration.OptionalString { + const wc = s.wc; + const result = try s.arena.alloc(Configuration.OptionalString, list.len); + for (result, list) |*dest, src| dest.* = try wc.addOptionalString(src); + return result; + } }; fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { @@ -320,7 +338,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { // Add and then de-duplicate dependencies. const deps = d: { const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); - for (try wc.prepareDeps(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| + for (try wc.reserveLengthPrefixed(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| dep.* = @intCast(step_map.getIndex(dep_step).?); break :d try wc.dedupeDeps(deps); }; @@ -340,11 +358,35 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .compile => e: { const c: *Step.Compile = @fieldParentPtr("step", step); + const exec_cmd_args: []const ?[]const u8 = c.exec_cmd_args orelse &.{}; + const installed_headers: []u32 = try arena.alloc(u32, c.installed_headers.items.len); + for (installed_headers, c.installed_headers.items) |*dst, src| switch (src) { + .file => |file| { + dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.File, .{ + .source = try s.addLazyPath(file.source), + .dest_sub_path = try wc.addString(file.dest_rel_path), + })); + }, + .directory => |directory| { + const include_extensions = directory.options.include_extensions orelse &.{}; + dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.Directory, .{ + .flags = .{ + .include_extensions = include_extensions.len != 0, + .exclude_extensions = directory.options.exclude_extensions.len != 0, + }, + .source = try s.addLazyPath(directory.source), + .dest_sub_path = try wc.addString(directory.dest_rel_path), + .exclude_extensions = .{ .slice = try s.initStringList(directory.options.exclude_extensions) }, + .include_extensions = .{ .slice = try s.initStringList(include_extensions) }, + })); + }, + }; + const extra_index = try wc.addExtra(@as(Configuration.Step.Compile, .{ .flags = .{ .filters_len = c.filters.len != 0, - .exec_cmd_args_len = if (c.exec_cmd_args) |a| a.len != 0 else false, - .installed_headers_len = c.installed_headers.items.len != 0, + .exec_cmd_args_len = exec_cmd_args.len != 0, + .installed_headers_len = installed_headers.len != 0, .force_undefined_symbols_len = c.force_undefined_symbols.entries.len != 0, .verbose_link = c.verbose_link, @@ -466,6 +508,10 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .hexstring => |*hexstring| try wc.addString(hexstring.toSlice()), .none, .fast, .uuid, .sha1, .md5 => null, } else null }, + .filters = .{ .slice = try s.initStringList(c.filters) }, + .exec_cmd_args = .{ .slice = try s.initOptionalStringList(exec_cmd_args) }, + .installed_headers = .initErased(installed_headers), + .force_undefined_symbols = .{ .slice = try s.initStringList(c.force_undefined_symbols.keys()) }, })); log.err("TODO serialize the trailing Compile step data", .{}); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 6cb1145351..069154a34e 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -33,7 +33,9 @@ pub const Header = extern struct { pub const Wip = struct { gpa: Allocator, string_table: StringTable = .empty, - deps_table: DepsTable = .empty, + /// De-duplicates an array inside `extra` that has first element length + /// followed by length elements. + length_prefixed_table: LengthPrefixedTable = .empty, targets_table: TargetsTable = .empty, string_bytes: std.ArrayList(u8) = .empty, @@ -44,23 +46,23 @@ pub const Wip = struct { path_deps: std.MultiArrayList(Path) = .empty, extra: std.ArrayList(u32) = .empty, - const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage); + const LengthPrefixedTable = std.HashMapUnmanaged(u32, void, LengthPrefixedContext, std.hash_map.default_max_load_percentage); const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage); - const DepsTableContext = struct { + const LengthPrefixedContext = struct { extra: []const u32, - pub fn eql(ctx: @This(), a: Deps, b: Deps) bool { - const len_a = ctx.extra[@intFromEnum(a)]; - const len_b = ctx.extra[@intFromEnum(b)]; - const slice_a = ctx.extra[@intFromEnum(a) + 1 ..][0..len_a]; - const slice_b = ctx.extra[@intFromEnum(b) + 1 ..][0..len_b]; + pub fn eql(ctx: @This(), a: u32, b: u32) bool { + const len_a = ctx.extra[a]; + const len_b = ctx.extra[b]; + const slice_a = ctx.extra[a + 1 ..][0..len_a]; + const slice_b = ctx.extra[b + 1 ..][0..len_b]; return std.mem.eql(u32, slice_a, slice_b); } - pub fn hash(ctx: @This(), key: Deps) u64 { - const len = ctx.extra[@intFromEnum(key)]; - const slice = ctx.extra[@intFromEnum(key) + 1 ..][0..len]; + pub fn hash(ctx: @This(), key: u32) u64 { + const len = ctx.extra[key]; + const slice = ctx.extra[key + 1 ..][0..len]; return std.hash_map.hashString(@ptrCast(slice)); } }; @@ -174,6 +176,10 @@ pub const Wip = struct { return new_off; } + pub fn addOptionalString(wip: *Wip, bytes: ?[]const u8) Allocator.Error!OptionalString { + return .init(try addString(wip, bytes orelse return .none)); + } + pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String { var buffer: [256]u8 = undefined; var writer: std.Io.Writer = .fixed(&buffer); @@ -324,27 +330,32 @@ pub const Wip = struct { } } - pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 { + pub fn reserveLengthPrefixed(wip: *Wip, n: usize) Allocator.Error![]u32 { const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); slice[0] = @intCast(n); return slice[1..]; } - pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { + pub fn dedupeLengthPrefixed(wip: *Wip, index: u32) Allocator.Error!u32 { + assert(wip.extra.items.len == index + wip.extra.items[index] + 1); const gpa = wip.gpa; - const gop = try wip.deps_table.getOrPutContext(gpa, deps, @as(DepsTableContext, .{ + const gop = try wip.length_prefixed_table.getOrPutContext(gpa, index, @as(LengthPrefixedContext, .{ .extra = wip.extra.items, })); if (gop.found_existing) { - wip.extra.items.len = @intFromEnum(deps); + wip.extra.items.len = index; return gop.key_ptr.*; } else { - return deps; + return index; } } + pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { + return @enumFromInt(try dedupeLengthPrefixed(wip, @intFromEnum(deps))); + } + pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { - const extra_len = Storage.calculateExtraLenUpperBound(@TypeOf(extra)); + const extra_len = Storage.extraLen(extra); try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len); return addExtraAssumeCapacity(wip, extra); } @@ -397,7 +408,7 @@ pub const Step = extern struct { owner: Package.Index, deps: Deps, max_rss: MaxRss, - extended: Storage.ExtendedIndex(Flags, union(Tag) { + extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, check_object: CheckObject, compile: Compile, @@ -585,10 +596,10 @@ pub const Step = extern struct { root_module: Module.Index, root_name: String, - //filters: FlagLengthPrefixedList(.flags, .filters_len, String), - //exec_cmd_args: FlagLengthPrefixedList(.flags, .exec_cmd_args_len, u32), - //installed_headers: FlagLengthPrefixedList(.flags, .installed_headers_len, InstalledHeader), - //force_undefined_symbols: FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), + filters: Storage.FlagLengthPrefixedList(.flags, .filters_len, String), + exec_cmd_args: Storage.FlagLengthPrefixedList(.flags, .exec_cmd_args_len, OptionalString), + installed_headers: Storage.FlagLengthPrefixedList(.flags, .installed_headers_len, Storage.Extended(InstalledHeader.Flags, InstalledHeader)), + force_undefined_symbols: Storage.FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), //exacts: EnumConditionalPrefixedList(.flags4, .expect_errors, .exact, String), linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath), version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath), @@ -612,6 +623,46 @@ pub const Step = extern struct { error_limit: Storage.FlagOptional(.flags4, .error_limit, u32), build_id: Storage.EnumOptional(.flags3, .build_id, .hexstring, String), + pub const InstalledHeader = union(@This().Tag) { + file: File, + directory: Directory, + + pub const Flags = packed struct(u32) { + tag: InstalledHeader.Tag, + _: u24 = 0, + }; + + pub const Tag = enum(u8) { + file, + directory, + }; + + pub const File = struct { + flags: @This().Flags = .{}, + source: LazyPath, + dest_sub_path: String, + + pub const Flags = packed struct(u32) { + tag: InstalledHeader.Tag = .file, + _: u24 = 0, + }; + }; + + pub const Directory = struct { + flags: @This().Flags, + source: LazyPath, + dest_sub_path: String, + exclude_extensions: Storage.FlagLengthPrefixedList(.flags, .exclude_extensions, String), + include_extensions: Storage.FlagLengthPrefixedList(.flags, .include_extensions, String), + + pub const Flags = packed struct(u32) { + tag: InstalledHeader.Tag = .directory, + exclude_extensions: bool, + include_extensions: bool, + _: u22 = 0, + }; + }; + }; pub const ExpectErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; pub const TestRunnerMode = enum(u2) { default, simple, server }; pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; @@ -1636,6 +1687,7 @@ pub const Storage = enum { flag_optional, enum_optional, extended, + flag_length_prefixed_list, /// The presence of the field is determined by a boolean within a packed /// struct. @@ -1674,16 +1726,7 @@ pub const Storage = enum { /// The field indexes into an auxilary buffer, with the first element being /// a packed struct that contains the tag. - pub fn Extended(comptime U: type) type { - return struct { - value: U, - - pub const storage: Storage = .extended; - }; - } - - /// Equivalent to `Extended` but works in an `extern struct`. - pub fn ExtendedIndex(comptime BaseFlags: type, comptime U: type) type { + pub fn Extended(comptime BaseFlags: type, comptime U: type) type { return enum(u32) { _, @@ -1699,6 +1742,30 @@ pub const Storage = enum { }; } + /// A field in flags determines whether the length is zero or nonzero. If the length is + /// nonzero, then there is a length field followed by the list. + /// + /// When deserializing, the slice field is set. When serializing, the index + /// field must be set. + pub fn FlagLengthPrefixedList( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime ValueArg: type, + ) type { + return struct { + slice: []const Value, + + pub const storage: Storage = .flag_length_prefixed_list; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const Value = ValueArg; + + pub fn initErased(s: []const u32) @This() { + return .{ .slice = @ptrCast(s) }; + } + }; + } + pub fn dataLength(buffer: []const u32, i: usize, comptime S: type) usize { var end = i; _ = data(buffer, &end, S); @@ -1769,6 +1836,15 @@ pub const Storage = enum { }; }, .extended => @compileError("TODO"), + .flag_length_prefixed_list => { + const flags = @field(container, @tagName(Field.flags)); + const flag = @field(flags, @tagName(Field.flag)); + if (!flag) return .{ .slice = &.{} }; + const data_start = i.* + 1; + const len = buffer[data_start - 1]; + defer i.* = data_start + len; + return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; + }, }, }, .@"extern" => comptime unreachable, @@ -1787,11 +1863,36 @@ pub const Storage = enum { return i; } - fn calculateExtraLenUpperBound(comptime Extra: type) comptime_int { - var i = 0; - const fields = @typeInfo(Extra).@"struct".fields; + fn extraFieldLen(field: anytype) usize { + const Field = @TypeOf(field); + return switch (@typeInfo(Field)) { + .int => |info| switch (info.bits) { + 32 => 1, + 64 => 2, + else => comptime unreachable, + }, + .@"enum" => 1, + .@"struct" => |info| switch (info.layout) { + .@"packed" => switch (info.backing_integer.?) { + u32 => 1, + u64 => 2, + else => comptime unreachable, + }, + .auto => switch (Field.storage) { + .flag_optional, .enum_optional, .extended => 1, + .flag_length_prefixed_list => field.slice.len + 1, + }, + .@"extern" => comptime unreachable, + }, + else => @compileError("bad type: " ++ @typeName(Field)), + }; + } + + fn extraLen(extra: anytype) usize { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + var i: usize = 0; inline for (fields) |field| { - i += calculateExtraFieldLenUpperBound(field.type); + i += Storage.extraFieldLen(@field(extra, field.name)); } return i; } @@ -1836,6 +1937,13 @@ pub const Storage = enum { return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0; }, .extended => @compileError("TODO"), + .flag_length_prefixed_list => { + const len: u32 = @intCast(value.slice.len); + if (len == 0) return 0; + buffer[i] = len; + @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); + return len + 1; + }, }, }, .@"extern" => comptime unreachable, @@ -1843,29 +1951,6 @@ pub const Storage = enum { else => @compileError("bad field type: " ++ @typeName(Field)), } } - - fn calculateExtraFieldLenUpperBound(comptime Field: type) comptime_int { - return switch (@typeInfo(Field)) { - .int => |info| switch (info.bits) { - 32 => 1, - 64 => 2, - else => comptime unreachable, - }, - .@"enum" => 1, - .@"struct" => |info| switch (info.layout) { - .@"packed" => switch (info.backing_integer.?) { - u32 => 1, - u64 => 2, - else => comptime unreachable, - }, - .auto => switch (Field.storage) { - .flag_optional, .enum_optional, .extended => 1, - }, - .@"extern" => comptime unreachable, - }, - else => comptime unreachable, - }; - } }; pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T {