diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig index 5e27c0bb29..838272ddaa 100644 --- a/lib/compiler/maker.zig +++ b/lib/compiler/maker.zig @@ -510,12 +510,10 @@ pub fn main(init: process.Init.Minimal) !void { else => |e| return e, }; - if (true) @panic("TODO"); - var w: Watch = w: { if (!watch) break :w undefined; if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(graph.cache.cwd); + break :w try .init(graph.cache.cwd, &scanned_config.configuration, run.steps); }; const now = Io.Clock.Timestamp.now(io, .awake); @@ -530,6 +528,7 @@ pub fn main(init: process.Init.Minimal) !void { .watch = watch, .listen_address = listen_address, .base_timestamp = now, + .configuration = &scanned_config.configuration, }); } else null; @@ -544,7 +543,7 @@ pub fn main(init: process.Init.Minimal) !void { }) { if (run.web_server) |*ws| ws.startBuild(); - try run.makeStepNames(step_names, main_progress_node, fuzz); + try run.makeStepNames(step_names.items, main_progress_node, fuzz); if (run.web_server) |*web_server| { if (fuzz) |mode| if (mode != .forever) fatal( @@ -556,12 +555,15 @@ pub fn main(init: process.Init.Minimal) !void { } if (run.web_server) |*ws| { + const c = &scanned_config.configuration; assert(!watch); // fatal error after CLI parsing while (true) switch (try ws.wait()) { .rebuild => { - for (run.step_stack.keys()) |step| { + for (run.step_stack.keys()) |step_index| { + const step = run.stepByIndex(step_index); step.state = .precheck_done; - step.pending_deps = @intCast(step.dependencies.items.len); + const deps = step_index.ptr(c).deps.slice(c); + step.pending_deps = @intCast(deps.len); step.reset(gpa); } continue :rebuild; @@ -581,7 +583,7 @@ pub fn main(init: process.Init.Minimal) !void { // recursive dependants. var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(run.step_stack.keys()), + w.dir_count, countSubProcesses(run.steps, run.step_stack.keys()), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); var in_debounce = false; @@ -589,7 +591,7 @@ pub fn main(init: process.Init.Minimal) !void { .timeout => { assert(in_debounce); debouncing_node.end(); - markFailedStepsDirty(gpa, run.step_stack.keys()); + markFailedStepsDirty(gpa, run.steps, run.step_stack.keys()); continue :rebuild; }, .dirty => if (!in_debounce) { @@ -602,22 +604,29 @@ pub fn main(init: process.Init.Minimal) !void { } } -fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { - for (all_steps) |step| switch (step.state) { - .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), - else => continue, - }; +fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), + else => continue, + } + } // Now that all dirty steps have been found, the remaining steps that // succeeded from last run shall be marked "cached". - for (all_steps) |step| switch (step.state) { - .success => step.result_cached = true, - else => continue, - }; + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .success => step.result_cached = true, + else => continue, + } + } } -fn countSubProcesses(all_steps: []const *Step) usize { +fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { var count: usize = 0; - for (all_steps) |s| { + for (all_steps) |step_index| { + const s = &make_steps[@intFromEnum(step_index)]; count += @intFromBool(s.getZigProcess() != null); } return count; @@ -703,7 +712,7 @@ const Run = struct { if (run.skip_oom_steps) { make_step.state = .skipped_oom; for (make_step.dependants.items) |dependant| { - dependant.pending_deps -= 1; + run.stepByIndex(dependant).pending_deps -= 1; } } else { log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ @@ -735,17 +744,19 @@ const Run = struct { const io = graph.io; const step_stack = &run.step_stack; const top_level_steps = &run.scanned_config.top_level_steps; + const c = &run.scanned_config.configuration; { // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking // a step is initial when it actually became ready due to an earlier initial step. - var initial_set: std.ArrayList(*Step) = .empty; + var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; defer initial_set.deinit(gpa); try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); - for (step_stack.keys()) |s| { + for (step_stack.keys()) |step_index| { + const s = run.stepByIndex(step_index); if (s.state == .precheck_done and s.pending_deps == 0) { - initial_set.appendAssumeCapacity(s); + initial_set.appendAssumeCapacity(step_index); } } @@ -755,7 +766,7 @@ const Run = struct { var group: Io.Group = .init; defer group.cancel(io); // Start working on all of the initial steps... - for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); + for (initial_set.items) |step_index| try stepReady(run, &group, step_index, step_prog); // ...and `makeStep` will trigger every other step when their last dependency finishes. try group.await(io); } @@ -779,16 +790,17 @@ const Run = struct { var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); defer cleanup_task.await(io); - for (step_stack.keys()) |s| { - test_pass_count += s.test_results.passCount(); - test_skip_count += s.test_results.skip_count; - test_fail_count += s.test_results.fail_count; - test_crash_count += s.test_results.crash_count; - test_timeout_count += s.test_results.timeout_count; + for (step_stack.keys()) |step_index| { + const make_step = run.stepByIndex(step_index); + test_pass_count += make_step.test_results.passCount(); + test_skip_count += make_step.test_results.skip_count; + test_fail_count += make_step.test_results.fail_count; + test_crash_count += make_step.test_results.crash_count; + test_timeout_count += make_step.test_results.timeout_count; - test_count += s.test_results.test_count; + test_count += make_step.test_results.test_count; - switch (s.state) { + switch (make_step.state) { .precheck_unstarted => unreachable, .precheck_started => unreachable, .precheck_done => unreachable, @@ -797,7 +809,7 @@ const Run = struct { .skipped, .skipped_oom => skipped_count += 1, .failure => { failure_count += 1; - const compile_errors_len = s.result_error_bundle.errorMessageCount(); + const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); if (compile_errors_len > 0) { total_compile_errors += compile_errors_len; } @@ -918,13 +930,17 @@ const Run = struct { var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { print_node.last = true; - printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(run, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; } else { const last_index = if (run.summary == .all) top_level_steps.count() else blk: { var i: usize = step_names.len; while (i > 0) { i -= 1; - const step = top_level_steps.get(step_names[i]).?.step; + const step_index = top_level_steps.get(step_names[i]).?; + const step = run.stepByIndex(step_index); const found = switch (run.summary) { .all, .line, .none => unreachable, .failures => step.state != .success, @@ -935,9 +951,12 @@ const Run = struct { break :blk top_level_steps.count(); }; for (step_names, 0..) |step_name, i| { - const tls = top_level_steps.get(step_name).?; + const step_index = top_level_steps.get(step_name).?; print_node.last = i + 1 == last_index; - printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(run, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; } } w.writeByte('\n') catch {}; @@ -957,120 +976,331 @@ const Run = struct { _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(code); } -}; -const PrintNode = struct { - parent: ?*PrintNode, - last: bool = false, -}; - -fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { - const parent = node.parent orelse return; - const writer = stderr.writer; - if (parent.parent == null) return; - try printPrefix(parent, stderr); - if (parent.last) { - try writer.writeAll(" "); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ - else => "| ", - }); + fn stepReady( + run: *Run, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, + ) Io.Cancelable!void { + const graph = run.graph; + const io = graph.io; + const c = &run.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + if (max_rss != 0) { + try run.max_rss_mutex.lock(io); + defer run.max_rss_mutex.unlock(io); + if (run.available_rss < max_rss) { + // Running this step right now could possibly exceed the allotted RSS. + run.memory_blocked_steps.append(run.gpa, step_index) catch + @panic("TODO eliminate memory allocation here"); + return; + } + run.available_rss -= max_rss; + } + group.async(io, makeStep, .{ run, group, step_index, root_prog_node }); } -} -fn printChildNodePrefix(stderr: Io.Terminal) !void { - try stderr.writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ - else => "+- ", - }); -} + /// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready + /// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must + /// have already subtracted this value from `run.available_rss`. This function will release the RSS + /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked + /// steps after "make" completes for `s`. + fn makeStep( + run: *Run, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, + ) Io.Cancelable!void { + const graph = run.graph; + const io = graph.io; + const gpa = run.gpa; + const c = &run.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const step_name = conf_step.name.slice(c); + const deps = conf_step.deps.slice(c); + const make_step = run.stepByIndex(step_index); -fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void { - const writer = stderr.writer; - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, + { + const step_prog_node = root_prog_node.start(step_name, 0); + defer step_prog_node.end(); - .dependency_failure => { - try stderr.setColor(.dim); - try writer.writeAll(" transitive failure\n"); - try stderr.setColor(.reset); - }, + if (run.web_server) |*ws| ws.updateStepStatus(step_index, .wip); - .success => { - try stderr.setColor(.green); - if (s.result_cached) { - try writer.writeAll(" cached"); - } else if (s.test_results.test_count > 0) { - const pass_count = s.test_results.passCount(); - assert(s.test_results.test_count == pass_count + s.test_results.skip_count); - try writer.print(" {d} pass", .{pass_count}); - if (s.test_results.skip_count > 0) { - try stderr.setColor(.reset); - try writer.writeAll(", "); - try stderr.setColor(.yellow); - try writer.print("{d} skip", .{s.test_results.skip_count}); + const new_state: Step.State = for (deps) |dep_index| { + const dep_make_step = run.stepByIndex(dep_index); + switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => break .dependency_failure, + + .success, .skipped => {}, } - try stderr.setColor(.reset); - try writer.print(" ({d} total)", .{s.test_results.test_count}); + } else if (make_step.make(.{ + .progress_node = step_prog_node, + .watch = run.watch, + .web_server = if (run.web_server) |*ws| ws else null, + .unit_test_timeout_ns = run.unit_test_timeout_ns, + .gpa = gpa, + })) state: { + break :state .success; + } else |err| switch (err) { + error.MakeFailed => .failure, + error.MakeSkipped => .skipped, + }; + + @atomicStore(Step.State, &make_step.state, new_state, .monotonic); + + switch (new_state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => { + if (run.web_server) |*ws| ws.updateStepStatus(step_index, .failure); + std.Progress.setStatus(.failure_working); + }, + + .success, + .skipped, + => { + if (run.web_server) |*ws| ws.updateStepStatus(step_index, .success); + }, + } + } + + // No matter the result, we want to display error/warning messages. + if (make_step.result_error_bundle.errorMessageCount() > 0 or + make_step.result_error_msgs.items.len > 0 or + make_step.result_stderr.len > 0) + { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + printErrorMessages(gpa, c, run.steps, step_index, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch |err| switch (err) { + error.Canceled => |e| return e, + error.WriteFailed => switch (stderr.file_writer.err.?) { + error.Canceled => |e| return e, + else => {}, + }, + else => {}, + }; + } + + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss != 0) { + var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer dispatch_set.deinit(gpa); + + // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` + // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. + { + try run.max_rss_mutex.lock(io); + defer run.max_rss_mutex.unlock(io); + run.available_rss += max_rss; + dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch + @panic("TODO eliminate memory allocation here"); + while (run.memory_blocked_steps.getLastOrNull()) |candidate_index| { + const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); + if (run.available_rss < candidate_max_rss) break; + assert(run.memory_blocked_steps.pop() == candidate_index); + dispatch_set.appendAssumeCapacity(candidate_index); + } + } + for (dispatch_set.items) |candidate| { + group.async(io, makeStep, .{ run, group, candidate, root_prog_node }); + } + } + + for (make_step.dependants.items) |dependant_index| { + const dependant = run.stepByIndex(dependant_index); + // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. + if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { + try stepReady(run, group, dependant_index, root_prog_node); + } + } + } + + fn printTreeStep( + run: *const Run, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + parent_node: *PrintNode, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + ) !void { + const writer = stderr.writer; + const first = step_stack.swapRemove(step_index); + const summary = run.summary; + const c = &run.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const make_step = run.stepByIndex(step_index); + const skip = switch (summary) { + .none, .line => unreachable, + .all => false, + .new => make_step.result_cached, + .failures => make_step.state == .success, + }; + if (skip) return; + try printPrefix(parent_node, stderr); + + if (parent_node.parent != null) { + if (parent_node.last) { + try printChildNodePrefix(stderr); } else { - try writer.writeAll(" success"); + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ + else => "+- ", + }); } - try stderr.setColor(.reset); - if (s.result_duration_ns) |ns| { - try stderr.setColor(.dim); - if (ns >= std.time.ns_per_min) { - try writer.print(" {d}m", .{ns / std.time.ns_per_min}); - } else if (ns >= std.time.ns_per_s) { - try writer.print(" {d}s", .{ns / std.time.ns_per_s}); - } else if (ns >= std.time.ns_per_ms) { - try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); - } else if (ns >= std.time.ns_per_us) { - try writer.print(" {d}us", .{ns / std.time.ns_per_us}); - } else { - try writer.print(" {d}ns", .{ns}); - } - try stderr.setColor(.reset); - } - if (s.result_peak_rss != 0) { - const rss = s.result_peak_rss; - try stderr.setColor(.dim); - if (rss >= 1000_000_000) { - try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); - } else if (rss >= 1000_000) { - try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); - } else if (rss >= 1000) { - try writer.print(" MaxRSS:{d}K", .{rss / 1000}); - } else { - try writer.print(" MaxRSS:{d}B", .{rss}); - } - try stderr.setColor(.reset); - } - try writer.writeAll("\n"); - }, - .skipped => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped\n"); - try stderr.setColor(.reset); - }, - .skipped_oom => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped (not enough memory)"); - try stderr.setColor(.dim); - try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss }); - try stderr.setColor(.reset); - }, - .failure => { - try printStepFailure(s, stderr, false); - try stderr.setColor(.reset); - }, - } -} + } -fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { + if (!first) try stderr.setColor(.dim); + + // dep_prefix omitted here because it is redundant with the tree. + try writer.writeAll(conf_step.name.slice(c)); + + const deps = conf_step.deps.slice(c); + + if (first) { + try printStepStatus(run, step_index, stderr); + + const last_index = if (summary == .all) deps.len -| 1 else blk: { + var i: usize = deps.len; + while (i > 0) { + i -= 1; + + const dep_index = deps[i]; + const dep = run.stepByIndex(dep_index); + const found = switch (summary) { + .all, .line, .none => unreachable, + .failures => dep.state != .success, + .new => !dep.result_cached, + }; + if (found) break :blk i; + } + break :blk deps.len -| 1; + }; + for (deps, 0..) |dep, i| { + var print_node: PrintNode = .{ + .parent = parent_node, + .last = i == last_index, + }; + try printTreeStep(run, dep, stderr, &print_node, step_stack); + } + } else { + if (deps.len == 0) { + try writer.writeAll(" (reused)\n"); + } else { + try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); + } + try stderr.setColor(.reset); + } + } + + fn printStepStatus(run: *const Run, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { + const s = run.stepByIndex(step_index); + const writer = stderr.writer; + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .dependency_failure => { + try stderr.setColor(.dim); + try writer.writeAll(" transitive failure\n"); + try stderr.setColor(.reset); + }, + + .success => { + try stderr.setColor(.green); + if (s.result_cached) { + try writer.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + assert(s.test_results.test_count == pass_count + s.test_results.skip_count); + try writer.print(" {d} pass", .{pass_count}); + if (s.test_results.skip_count > 0) { + try stderr.setColor(.reset); + try writer.writeAll(", "); + try stderr.setColor(.yellow); + try writer.print("{d} skip", .{s.test_results.skip_count}); + } + try stderr.setColor(.reset); + try writer.print(" ({d} total)", .{s.test_results.test_count}); + } else { + try writer.writeAll(" success"); + } + try stderr.setColor(.reset); + if (s.result_duration_ns) |ns| { + try stderr.setColor(.dim); + if (ns >= std.time.ns_per_min) { + try writer.print(" {d}m", .{ns / std.time.ns_per_min}); + } else if (ns >= std.time.ns_per_s) { + try writer.print(" {d}s", .{ns / std.time.ns_per_s}); + } else if (ns >= std.time.ns_per_ms) { + try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); + } else if (ns >= std.time.ns_per_us) { + try writer.print(" {d}us", .{ns / std.time.ns_per_us}); + } else { + try writer.print(" {d}ns", .{ns}); + } + try stderr.setColor(.reset); + } + if (s.result_peak_rss != 0) { + const rss = s.result_peak_rss; + try stderr.setColor(.dim); + if (rss >= 1000_000_000) { + try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); + } else if (rss >= 1000_000) { + try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); + } else if (rss >= 1000) { + try writer.print(" MaxRSS:{d}K", .{rss / 1000}); + } else { + try writer.print(" MaxRSS:{d}B", .{rss}); + } + try stderr.setColor(.reset); + } + try writer.writeAll("\n"); + }, + .skipped => { + try stderr.setColor(.yellow); + try writer.writeAll(" skipped\n"); + try stderr.setColor(.reset); + }, + .skipped_oom => { + const c = &run.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + try stderr.setColor(.yellow); + try writer.writeAll(" skipped (not enough memory)"); + try stderr.setColor(.dim); + try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ + max_rss, run.available_rss, + }); + try stderr.setColor(.reset); + }, + .failure => { + try printStepFailure(run.steps, step_index, stderr, false); + try stderr.setColor(.reset); + }, + } + } +}; + +fn printStepFailure( + make_steps: []Step, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + dim: bool, +) !void { const w = stderr.writer; + const s = &make_steps[@intFromEnum(step_index)]; if (s.result_error_bundle.errorMessageCount() > 0) { try stderr.setColor(.red); try w.print(" {d} errors\n", .{ @@ -1153,79 +1383,33 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { } } -fn printTreeStep( - graph: *Graph, - s: *Step, - run: *const Run, - stderr: Io.Terminal, - parent_node: *PrintNode, - step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), -) !void { +const PrintNode = struct { + parent: ?*PrintNode, + last: bool = false, +}; + +fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { + const parent = node.parent orelse return; const writer = stderr.writer; - const first = step_stack.swapRemove(s); - const summary = run.summary; - const skip = switch (summary) { - .none, .line => unreachable, - .all => false, - .new => s.result_cached, - .failures => s.state == .success, - }; - if (skip) return; - try printPrefix(parent_node, stderr); - - if (parent_node.parent != null) { - if (parent_node.last) { - try printChildNodePrefix(stderr); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ - else => "+- ", - }); - } - } - - if (!first) try stderr.setColor(.dim); - - // dep_prefix omitted here because it is redundant with the tree. - try writer.writeAll(s.name); - - if (first) { - try printStepStatus(s, stderr, run); - - const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { - var i: usize = s.dependencies.items.len; - while (i > 0) { - i -= 1; - - const step = s.dependencies.items[i]; - const found = switch (summary) { - .all, .line, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk s.dependencies.items.len -| 1; - }; - for (s.dependencies.items, 0..) |dep, i| { - var print_node: PrintNode = .{ - .parent = parent_node, - .last = i == last_index, - }; - try printTreeStep(graph, dep, run, stderr, &print_node, step_stack); - } + if (parent.parent == null) return; + try printPrefix(parent, stderr); + if (parent.last) { + try writer.writeAll(" "); } else { - if (s.dependencies.items.len == 0) { - try writer.writeAll(" (reused)\n"); - } else { - try writer.print(" (+{d} more reused dependencies)\n", .{ - s.dependencies.items.len, - }); - } - try stderr.setColor(.reset); + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ + else => "| ", + }); } } +fn printChildNodePrefix(stderr: Io.Terminal) !void { + try stderr.writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ + else => "+- ", + }); +} + /// Traverse the dependency graph depth-first and make it undirected by having /// steps know their dependants (they only know dependencies at start). /// Along the way, check that there is no dependency loop, and record the steps @@ -1245,14 +1429,14 @@ fn constructGraphAndCheckForDependencyLoop( step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), rand: std.Random, ) error{ DependencyLoopDetected, OutOfMemory }!void { - const s: *Step = &steps[@intFromEnum(step_index)]; - switch (s.state) { + const make_step: *Step = &steps[@intFromEnum(step_index)]; + switch (make_step.state) { .precheck_started => { log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); return error.DependencyLoopDetected; }, .precheck_unstarted => { - s.state = .precheck_started; + make_step.state = .precheck_started; const step = step_index.ptr(c); const dependencies = step.deps.slice(c); @@ -1268,7 +1452,7 @@ fn constructGraphAndCheckForDependencyLoop( for (deps) |dep| { const dep_step: *Step = &steps[@intFromEnum(dep)]; try step_stack.put(gpa, dep, {}); - try dep_step.dependants.append(gpa, s); + try dep_step.dependants.append(gpa, step_index); constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { error.DependencyLoopDetected => { log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); @@ -1278,8 +1462,8 @@ fn constructGraphAndCheckForDependencyLoop( }; } - s.state = .precheck_done; - s.pending_deps = @intCast(dependencies.len); + make_step.state = .precheck_done; + make_step.pending_deps = @intCast(dependencies.len); }, .precheck_done => {}, @@ -1292,140 +1476,11 @@ fn constructGraphAndCheckForDependencyLoop( } } -/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready -/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must -/// have already subtracted this value from `run.available_rss`. This function will release the RSS -/// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked -/// steps after "make" completes for `s`. -fn makeStep( - graph: *Graph, - group: *Io.Group, - s: *Step, - root_prog_node: std.Progress.Node, - run: *Run, -) Io.Cancelable!void { - const io = graph.io; - const gpa = run.gpa; - - { - const step_prog_node = root_prog_node.start(s.name, 0); - defer step_prog_node.end(); - - if (run.web_server) |*ws| ws.updateStepStatus(s, .wip); - - const new_state: Step.State = for (s.dependencies.items) |dep| { - switch (@atomicLoad(Step.State, &dep.state, .monotonic)) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => break .dependency_failure, - - .success, .skipped => {}, - } - } else if (s.make(.{ - .progress_node = step_prog_node, - .watch = run.watch, - .web_server = if (run.web_server) |*ws| ws else null, - .unit_test_timeout_ns = run.unit_test_timeout_ns, - .gpa = gpa, - })) state: { - break :state .success; - } else |err| switch (err) { - error.MakeFailed => .failure, - error.MakeSkipped => .skipped, - }; - - @atomicStore(Step.State, &s.state, new_state, .monotonic); - - switch (new_state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => { - if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); - std.Progress.setStatus(.failure_working); - }, - - .success, - .skipped, - => { - if (run.web_server) |*ws| ws.updateStepStatus(s, .success); - }, - } - } - - // No matter the result, we want to display error/warning messages. - if (s.result_error_bundle.errorMessageCount() > 0 or - s.result_error_msgs.items.len > 0 or - s.result_stderr.len > 0) - { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {}; - } - - if (s.max_rss != 0) { - var dispatch_set: std.ArrayList(*Step) = .empty; - defer dispatch_set.deinit(gpa); - - // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` - // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. - { - try run.max_rss_mutex.lock(io); - defer run.max_rss_mutex.unlock(io); - run.available_rss += s.max_rss; - try dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len); - while (run.memory_blocked_steps.getLastOrNull()) |candidate| { - if (run.available_rss < candidate.max_rss) break; - assert(run.memory_blocked_steps.pop() == candidate); - dispatch_set.appendAssumeCapacity(candidate); - } - } - for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run }); - } - } - - for (s.dependants.items) |dependant| { - // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. - if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(graph, group, dependant, root_prog_node, run); - } - } -} - -fn stepReady( - graph: *Graph, - group: *Io.Group, - s: *Step, - root_prog_node: std.Progress.Node, - run: *Run, -) !void { - const io = graph.io; - if (s.max_rss != 0) { - try run.max_rss_mutex.lock(io); - defer run.max_rss_mutex.unlock(io); - if (run.available_rss < s.max_rss) { - // Running this step right now could possibly exceed the allotted RSS. - try run.memory_blocked_steps.append(run.gpa, s); - return; - } - run.available_rss -= s.max_rss; - } - group.async(io, makeStep, .{ graph, group, s, root_prog_node, run }); -} - pub fn printErrorMessages( gpa: Allocator, - failing_step: *Step, + c: *const Configuration, + make_steps: []Step, + failing_step_index: Configuration.Step.Index, options: std.zig.ErrorBundle.RenderOptions, stderr: Io.Terminal, error_style: ErrorStyle, @@ -1435,26 +1490,28 @@ pub fn printErrorMessages( if (error_style.verboseContext()) { // Provide context for where these error messages are coming from by // printing the corresponding Step subtree. - var step_stack: std.ArrayList(*Step) = .empty; + var step_stack: std.ArrayList(Configuration.Step.Index) = .empty; defer step_stack.deinit(gpa); - try step_stack.append(gpa, failing_step); - while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { - try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); + try step_stack.append(gpa, failing_step_index); + while (true) { + const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; + if (last_step.dependants.items.len == 0) break; + try step_stack.append(gpa, last_step.dependants.items[0]); } // Now, `step_stack` has the subtree that we want to print, in reverse order. try stderr.setColor(.dim); var indent: usize = 0; - while (step_stack.pop()) |s| : (indent += 1) { + while (step_stack.pop()) |step_index| : (indent += 1) { if (indent > 0) { try writer.splatByteAll(' ', (indent - 1) * 3); try printChildNodePrefix(stderr); } - try writer.writeAll(s.name); + try writer.writeAll(step_index.ptr(c).name.slice(c)); - if (s == failing_step) { - try printStepFailure(s, stderr, true); + if (step_index == failing_step_index) { + try printStepFailure(make_steps, step_index, stderr, true); } else { try writer.writeAll("\n"); } @@ -1463,11 +1520,13 @@ pub fn printErrorMessages( } else { // Just print the failing step itself. try stderr.setColor(.dim); - try writer.writeAll(failing_step.name); - try printStepFailure(failing_step, stderr, true); + try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); + try printStepFailure(make_steps, failing_step_index, stderr, true); try stderr.setColor(.reset); } + const failing_step = &make_steps[@intFromEnum(failing_step_index)]; + if (failing_step.result_stderr.len > 0) { try writer.writeAll(failing_step.result_stderr); if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { @@ -1560,9 +1619,10 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { fatal(f, args); } -fn cleanTmpFiles(io: Io, steps: []const *Step) void { - for (steps) |step| { - const wf = step.cast(Step.WriteFile) orelse continue; +fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { + for (steps) |step_index| { + if (true) @panic("TODO"); + const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; Io.Dir.cwd().deleteTree(io, path) catch |err| { diff --git a/lib/compiler/maker/Fuzz.zig b/lib/compiler/maker/Fuzz.zig index 2999b7dffc..f29eaca5b2 100644 --- a/lib/compiler/maker/Fuzz.zig +++ b/lib/compiler/maker/Fuzz.zig @@ -1,16 +1,16 @@ const Fuzz = @This(); const std = @import("std"); -const Io = std.Io; +const Allocator = std.mem.Allocator; const Build = std.Build; const Cache = std.Build.Cache; -const Step = std.Build.Step; +const Coverage = std.debug.Coverage; +const Configuration = std.Build.Configuration; +const Io = std.Io; +const abi = std.Build.abi.fuzz; const assert = std.debug.assert; const fatal = std.process.fatal; -const Allocator = std.mem.Allocator; const log = std.log; -const Coverage = std.debug.Coverage; -const abi = std.Build.abi.fuzz; const maker = @import("../maker.zig"); const WebServer = @import("WebServer.zig"); @@ -20,7 +20,7 @@ io: Io, mode: Mode, /// Allocated into `gpa`. -run_steps: []const *Step.Run, +run_steps: []const Configuration.Step.Index, group: Io.Group, root_prog_node: std.Progress.Node, @@ -51,7 +51,7 @@ const Msg = union(enum) { unique: u64, coverage: u64, }, - run: *Step.Run, + run: Configuration.Step.Index, }, entry_point: struct { coverage_id: u64, @@ -78,12 +78,12 @@ const CoverageMap = struct { pub fn init( gpa: Allocator, io: Io, - all_steps: []const *Build.Step, + all_steps: []const Configuration.Step.Index, root_prog_node: std.Progress.Node, mode: Mode, ) error{ OutOfMemory, Canceled }!Fuzz { - const run_steps: []const *Step.Run = steps: { - var steps: std.ArrayList(*Step.Run) = .empty; + const run_steps: []const Configuration.Step.Index = steps: { + var steps: std.ArrayList(Configuration.Step.Index) = .empty; defer steps.deinit(gpa); const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); defer rebuild_node.end(); @@ -91,7 +91,8 @@ pub fn init( defer rebuild_group.cancel(io); for (all_steps) |step| { - const run = step.cast(Step.Run) orelse continue; + if (true) @panic("TODO"); + const run = step.cast(std.Build.Step.Run) orelse continue; if (run.producer == null) continue; if (run.fuzz_tests.items.len == 0) continue; try steps.append(gpa, run); @@ -100,15 +101,16 @@ pub fn init( if (steps.items.len == 0) fatal("no fuzz tests found", .{}); rebuild_node.setEstimatedTotalItems(steps.items.len); - const run_steps = try gpa.dupe(*Step.Run, steps.items); + const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items); try rebuild_group.await(io); break :steps run_steps; }; errdefer gpa.free(run_steps); - for (run_steps) |run| { - assert(run.fuzz_tests.items.len > 0); - if (run.rebuilt_executable == null) + for (run_steps) |run_step_index| { + if (true) @panic("TODO"); + assert(run_step_index.fuzz_tests.items.len > 0); + if (run_step_index.rebuilt_executable == null) fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } @@ -147,6 +149,7 @@ pub fn start(fuzz: *Fuzz) void { } for (fuzz.run_steps) |run| { + if (true) @panic("TODO"); for (run.fuzz_tests.items) |unit_test_index| { assert(run.rebuilt_executable != null); fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index }); @@ -161,14 +164,14 @@ pub fn deinit(fuzz: *Fuzz) void { fuzz.gpa.free(fuzz.run_steps); } -fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) void { +fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { const compile = run.producer.?; log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); }; } -fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { +fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { const graph = run.step.owner.graph; const io = graph.io; const compile = run.producer.?; @@ -195,7 +198,7 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); } -fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void { +fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index, unit_test_index: u32) void { const owner = run.step.owner; const gpa = owner.allocator; const graph = owner.graph; @@ -223,6 +226,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void { } pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { + if (true) @panic("TODO"); assert(fuzz.mode == .forever); var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); @@ -368,7 +372,8 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { fuzz.msg_queue.clearRetainingCapacity(); } } -fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { +fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { + if (true) @panic("TODO"); assert(fuzz.mode == .forever); const ws = fuzz.mode.forever.ws; const gpa = fuzz.gpa; @@ -398,8 +403,8 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO }; errdefer gop.value_ptr.coverage.deinit(gpa); - const rebuilt_exe_path = run_step.rebuilt_executable.?; - const target = run_step.producer.?.rootModuleTarget(); + const rebuilt_exe_path = run_step_index.rebuilt_executable.?; + const target = run_step_index.producer.?.rootModuleTarget(); var debug_info = std.debug.Info.load( gpa, io, @@ -409,19 +414,19 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO target.cpu.arch, ) catch |err| { log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ - run_step.step.name, rebuilt_exe_path, err, + run_step_index.step.name, rebuilt_exe_path, err, }); return error.AlreadyReported; }; defer debug_info.deinit(gpa); const coverage_file_path: Build.Cache.Path = .{ - .root_dir = run_step.step.owner.cache_root, + .root_dir = run_step_index.step.owner.cache_root, .sub_path = "v/" ++ std.fmt.hex(coverage_id), }; var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ - run_step.step.name, coverage_file_path, err, + run_step_index.step.name, coverage_file_path, err, }); return error.AlreadyReported; }; @@ -528,6 +533,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte } pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { + if (true) @panic("TODO"); assert(fuzz.mode == .limit); const io = fuzz.io; diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig index 8243c37d8a..906bc810f7 100644 --- a/lib/compiler/maker/Step.zig +++ b/lib/compiler/maker/Step.zig @@ -10,6 +10,7 @@ const Io = std.Io; const LazyPath = std.Build.Configuration.LazyPath; const Package = std.Build.Configuration.Package; const Path = std.Build.Cache.Path; +const Configuration = std.Build.Configuration; const assert = std.debug.assert; const WebServer = @import("WebServer.zig"); @@ -21,7 +22,7 @@ pub const Run = void; // @import("Step/Run.zig"); _: void align(std.atomic.cache_line) = {}, state: State = .precheck_unstarted, -dependants: std.ArrayList(*Step) = .empty, +dependants: std.ArrayList(Configuration.Step.Index) = .empty, /// Collects the set of files that retrigger this step to run. /// /// This is used by the build system's implementation of `--watch` but it can @@ -143,6 +144,7 @@ pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; /// have already reported the error. Otherwise, we add a simple error report /// here. pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { + if (true) @panic("TODO Step.make"); const arena = s.owner.allocator; const graph = s.owner.graph; const io = graph.io; @@ -183,6 +185,7 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi /// Implementation detail of file watching. Prepares the step for being re-evaluated. /// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. pub fn invalidateResult(step: *Step, gpa: Allocator) bool { + if (true) @panic("TODO Step.invalidateResult"); if (step.state == .precheck_done) return false; assert(step.pending_deps == 0); step.state = .precheck_done; @@ -544,6 +547,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer } pub fn getZigProcess(s: *Step) ?*ZigProcess { + if (true) @panic("TODO getZigProcess"); return switch (s.id) { .compile => s.cast(Compile).?.zig_process, else => null, diff --git a/lib/compiler/maker/Watch.zig b/lib/compiler/maker/Watch.zig index 57f0c5d403..5e52457476 100644 --- a/lib/compiler/maker/Watch.zig +++ b/lib/compiler/maker/Watch.zig @@ -3,11 +3,13 @@ const builtin = @import("builtin"); const std = @import("std"); const Io = std.Io; -const Step = std.Build.Step; const Allocator = std.mem.Allocator; const assert = std.debug.assert; const fatal = std.process.fatal; +const Configuration = std.Build.Configuration; + const FsEvents = @import("Watch/FsEvents.zig"); +const Step = @import("Step.zig"); os: Os, /// The number to show as the number of directories being watched. @@ -16,6 +18,8 @@ dir_count: usize, // They are `undefined` on implementations which do not utilize then. dir_table: DirTable, generation: Generation, +configuration: *const Configuration, +make_steps: []Step, pub const have_impl = Os != void; @@ -27,7 +31,7 @@ const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAda /// Special key of "." means any changes in this directory trigger the steps. const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); -const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); +const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation); const Generation = u8; @@ -101,7 +105,7 @@ const Os = switch (builtin.os.tag) { }; }; - fn init(cwd_path: []const u8) !Watch { + fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { _ = cwd_path; return .{ .dir_table = .{}, @@ -114,6 +118,8 @@ const Os = switch (builtin.os.tag) { else => {}, }, .generation = 0, + .make_steps = make_steps, + .configuration = configuration, }; } @@ -161,20 +167,21 @@ const Os = switch (builtin.os.tag) { const lfh: FileHandle = .{ .handle = file_handle }; if (w.os.handle_table.getPtr(lfh)) |value| { if (value.reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty); if (value.reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty); } }, - else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), + else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}), } } } } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { // Add missing marks and note persisted ones. - for (steps) |step| { + for (steps) |step_index| { + const step = &w.make_steps[@intFromEnum(step_index)]; for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); @@ -236,7 +243,7 @@ const Os = switch (builtin.os.tag) { for (files.items) |basename| { const gop = try reaction_set.getOrPut(gpa, basename); if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); + try gop.value_ptr.put(gpa, step_index, w.generation); } } } @@ -537,7 +544,7 @@ const Os = switch (builtin.os.tag) { return any_dirty; } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { // Add missing marks and note persisted ones. for (steps) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { @@ -678,7 +685,7 @@ const Os = switch (builtin.os.tag) { }; } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { const handles = &w.os.handles; for (steps) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { @@ -856,7 +863,7 @@ const Os = switch (builtin.os.tag) { .generation = undefined, }; } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { try w.os.fse.setPaths(gpa, steps); w.dir_count = w.os.fse.watch_roots.len; } @@ -871,8 +878,8 @@ const Os = switch (builtin.os.tag) { else => void, }; -pub fn init(cwd_path: []const u8) !Watch { - return Os.init(cwd_path); +pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { + return Os.init(cwd_path, configuration, make_steps); } pub const Match = struct { @@ -880,20 +887,19 @@ pub const Match = struct { /// match. basename: []const u8, /// The step to re-run when file corresponding to `basename` is changed. - step: *Step, + step_index: Configuration.Step.Index, pub const Context = struct { pub fn hash(self: Context, a: Match) u32 { _ = self; - var hasher = Hash.init(0); - std.hash.autoHash(&hasher, a.step); + var hasher = Hash.init(@intFromEnum(a.step_index)); hasher.update(a.basename); return @truncate(hasher.final()); } pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { _ = self; _ = b_index; - return a.step == b.step and std.mem.eql(u8, a.basename, b.basename); + return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename); } }; }; @@ -908,22 +914,24 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { else => item, }; for (reaction_set.values()) |step_set| { - for (step_set.keys()) |step| { + for (step_set.keys()) |step_index| { + const step = &w.make_steps[@intFromEnum(step_index)]; _ = step.invalidateResult(gpa); } } } } -fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool { +fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool { var this_any_dirty = false; - for (step_set.keys()) |step| { + for (step_set.keys()) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; if (step.invalidateResult(gpa)) this_any_dirty = true; } return any_dirty or this_any_dirty; } -pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { +pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { return Os.update(w, gpa, steps); } diff --git a/lib/compiler/maker/WebServer.zig b/lib/compiler/maker/WebServer.zig index 501bed6ed6..16f6b05c8f 100644 --- a/lib/compiler/maker/WebServer.zig +++ b/lib/compiler/maker/WebServer.zig @@ -4,8 +4,8 @@ const builtin = @import("builtin"); const std = @import("std"); const Allocator = std.mem.Allocator; -const Build = std.Build; const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; const Io = std.Io; const abi = std.Build.abi; const assert = std.debug.assert; @@ -15,10 +15,12 @@ const mem = std.mem; const net = std.Io.net; const Fuzz = @import("Fuzz.zig"); +const Graph = @import("Graph.zig"); +const Step = @import("Step.zig"); gpa: Allocator, -graph: *const Build.Graph, -all_steps: []const *Build.Step, +graph: *const Graph, +all_steps: []const Configuration.Step.Index, listen_address: net.IpAddress, root_prog_node: std.Progress.Node, watch: bool, @@ -69,12 +71,13 @@ pub fn notifyUpdate(ws: *WebServer) void { pub const Options = struct { gpa: Allocator, - graph: *const std.Build.Graph, - all_steps: []const *Build.Step, + graph: *const Graph, + all_steps: []const Configuration.Step.Index, root_prog_node: std.Progress.Node, watch: bool, listen_address: net.IpAddress, base_timestamp: Io.Clock.Timestamp, + configuration: *const Configuration, }; pub fn init(opts: Options) WebServer { // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` @@ -83,19 +86,21 @@ pub fn init(opts: Options) WebServer { assert(opts.base_timestamp.clock == base_clock); const all_steps = opts.all_steps; + const c = opts.configuration; const step_names_trailing = opts.gpa.alloc(u8, len: { var name_bytes: usize = 0; - for (all_steps) |step| name_bytes += step.name.len; + for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len; break :len name_bytes + all_steps.len * 4; }) catch @panic("out of memory"); { const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); var idx: usize = all_steps.len * 4; - for (all_steps, step_name_lens) |step, *name_len| { - name_len.* = @intCast(step.name.len); - @memcpy(step_names_trailing[idx..][0..step.name.len], step.name); - idx += step.name.len; + for (all_steps, step_name_lens) |step_index, *name_len| { + const step_name = step_index.ptr(c).name.slice(c); + name_len.* = @intCast(step_name.len); + @memcpy(step_names_trailing[idx..][0..step_name.len], step_name); + idx += step_name.len; } assert(idx == step_names_trailing.len); } @@ -208,9 +213,14 @@ pub fn startBuild(ws: *WebServer) void { ws.notifyUpdate(); } -pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void { +pub fn updateStepStatus( + ws: *WebServer, + step_index: Configuration.Step.Index, + new_status: abi.StepUpdate.Status, +) void { + // TODO don't do linear search, especially in a hot loop like this const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == step) break @intCast(i); + if (s == step_index) break @intCast(i); } else unreachable; const ptr = &ws.step_status_bits[step_idx / 4]; const bit_offset: u3 = @intCast((step_idx % 4) * 2); @@ -673,7 +683,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim if (code != 0) { log.err( "the following command exited with error code {d}:\n{s}", - .{ code, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; } @@ -681,14 +691,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim .signal => |sig| { log.err( "the following command terminated with signal {t}:\n{s}", - .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; }, .stopped, .unknown => { log.err( "the following command terminated unexpectedly:\n{s}", - .{try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)}, + .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)}, ); return error.WasmCompilationFailed; }, @@ -698,14 +708,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim try result_error_bundle.renderToStderr(io, .{}, .auto); log.err("the following command failed with {d} compilation errors:\n{s}", .{ result_error_bundle.errorMessageCount(), - try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), + try Step.allocPrintCmd(arena, .inherit, null, argv.items), }); return error.WasmCompilationFailed; } const base_path = result orelse { log.err("child process failed to report result\n{s}", .{ - try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), + try Step.allocPrintCmd(arena, .inherit, null, argv.items), }); return error.WasmCompilationFailed; }; @@ -729,7 +739,7 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 } pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { - compile: *Build.Step.Compile, + compile_step: Configuration.Step.Index, use_llvm: bool, stats: abi.time_report.CompileResult.Stats, @@ -745,8 +755,9 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { const gpa = ws.gpa; const io = ws.graph.io; + // TODO don't do linear search const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == &opts.compile.step) break @intCast(i); + if (s == opts.compile_step) break @intCast(i); } else unreachable; const old_buf = old: { @@ -782,12 +793,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { ws.notifyUpdate(); } -pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.Duration) void { +pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void { const gpa = ws.gpa; const io = ws.graph.io; + // TODO don't do linear search const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == step) break @intCast(i); + if (s == step_index) break @intCast(i); } else unreachable; const old_buf = old: { @@ -815,15 +827,16 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.D pub fn updateTimeReportRunTest( ws: *WebServer, - run: *Build.Step.Run, - tests: *const Build.Step.Run.CachedTestMetadata, + run_step_index: Configuration.Step.Index, + tests: *const Step.Run.CachedTestMetadata, ns_per_test: []const u64, ) void { const gpa = ws.gpa; const io = ws.graph.io; + // TODO don't do linear search const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == &run.step) break @intCast(i); + if (s == run_step_index) break @intCast(i); } else unreachable; assert(tests.names.len == ns_per_test.len);