archives/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2026-03-08 03:44:46 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

make runner: execute step graph

Browse source
This commit is contained in:
Andrew Kelley 2026-02-18 13:12:18 -08:00
parent 4fa061cc6a
commit ef874f446d
5 changed files with 521 additions and 430 deletions
Download patch file Download diff file Expand all files Collapse all files

778
lib/compiler/maker.zig
View file

@ -510,12 +510,10 @@ pub fn main(init: process.Init.Minimal) !void {
else => |e| return e,
};
if (true) @panic("TODO");
var w: Watch = w: {
if (!watch) break :w undefined;
if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag});
break :w try .init(graph.cache.cwd);
break :w try .init(graph.cache.cwd, &scanned_config.configuration, run.steps);
};
const now = Io.Clock.Timestamp.now(io, .awake);
@ -530,6 +528,7 @@ pub fn main(init: process.Init.Minimal) !void {
.watch = watch,
.listen_address = listen_address,
.base_timestamp = now,
.configuration = &scanned_config.configuration,
});
} else null;
@ -544,7 +543,7 @@ pub fn main(init: process.Init.Minimal) !void {
}) {
if (run.web_server) |*ws| ws.startBuild();
try run.makeStepNames(step_names, main_progress_node, fuzz);
try run.makeStepNames(step_names.items, main_progress_node, fuzz);
if (run.web_server) |*web_server| {
if (fuzz) |mode| if (mode != .forever) fatal(
@ -556,12 +555,15 @@ pub fn main(init: process.Init.Minimal) !void {
}
if (run.web_server) |*ws| {
const c = &scanned_config.configuration;
assert(!watch); // fatal error after CLI parsing
while (true) switch (try ws.wait()) {
.rebuild => {
for (run.step_stack.keys()) |step| {
for (run.step_stack.keys()) |step_index| {
const step = run.stepByIndex(step_index);
step.state = .precheck_done;
step.pending_deps = @intCast(step.dependencies.items.len);
const deps = step_index.ptr(c).deps.slice(c);
step.pending_deps = @intCast(deps.len);
step.reset(gpa);
}
continue :rebuild;
@ -581,7 +583,7 @@ pub fn main(init: process.Init.Minimal) !void {
// recursive dependants.
var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
w.dir_count, countSubProcesses(run.step_stack.keys()),
w.dir_count, countSubProcesses(run.steps, run.step_stack.keys()),
}) catch &caption_buf;
var debouncing_node = main_progress_node.start(caption, 0);
var in_debounce = false;
@ -589,7 +591,7 @@ pub fn main(init: process.Init.Minimal) !void {
.timeout => {
assert(in_debounce);
debouncing_node.end();
markFailedStepsDirty(gpa, run.step_stack.keys());
markFailedStepsDirty(gpa, run.steps, run.step_stack.keys());
continue :rebuild;
},
.dirty => if (!in_debounce) {
@ -602,22 +604,29 @@ pub fn main(init: process.Init.Minimal) !void {
}
}
fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {
for (all_steps) |step| switch (step.state) {
.dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),
else => continue,
};
fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void {
for (all_steps) |step_index| {
const step = &make_steps[@intFromEnum(step_index)];
switch (step.state) {
.dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),
else => continue,
}
}
// Now that all dirty steps have been found, the remaining steps that
// succeeded from last run shall be marked "cached".
for (all_steps) |step| switch (step.state) {
.success => step.result_cached = true,
else => continue,
};
for (all_steps) |step_index| {
const step = &make_steps[@intFromEnum(step_index)];
switch (step.state) {
.success => step.result_cached = true,
else => continue,
}
}
}
fn countSubProcesses(all_steps: []const *Step) usize {
fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize {
var count: usize = 0;
for (all_steps) |s| {
for (all_steps) |step_index| {
const s = &make_steps[@intFromEnum(step_index)];
count += @intFromBool(s.getZigProcess() != null);
}
return count;
@ -703,7 +712,7 @@ const Run = struct {
if (run.skip_oom_steps) {
make_step.state = .skipped_oom;
for (make_step.dependants.items) |dependant| {
dependant.pending_deps -= 1;
run.stepByIndex(dependant).pending_deps -= 1;
}
} else {
log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
@ -735,17 +744,19 @@ const Run = struct {
const io = graph.io;
const step_stack = &run.step_stack;
const top_level_steps = &run.scanned_config.top_level_steps;
const c = &run.scanned_config.configuration;
{
// Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,
// then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking
// a step is initial when it actually became ready due to an earlier initial step.
var initial_set: std.ArrayList(*Step) = .empty;
var initial_set: std.ArrayList(Configuration.Step.Index) = .empty;
defer initial_set.deinit(gpa);
try initial_set.ensureUnusedCapacity(gpa, step_stack.count());
for (step_stack.keys()) |s| {
for (step_stack.keys()) |step_index| {
const s = run.stepByIndex(step_index);
if (s.state == .precheck_done and s.pending_deps == 0) {
initial_set.appendAssumeCapacity(s);
initial_set.appendAssumeCapacity(step_index);
}
}
@ -755,7 +766,7 @@ const Run = struct {
var group: Io.Group = .init;
defer group.cancel(io);
// Start working on all of the initial steps...
for (initial_set.items) |s| try stepReady(&group, s, step_prog, run);
for (initial_set.items) |step_index| try stepReady(run, &group, step_index, step_prog);
// ...and `makeStep` will trigger every other step when their last dependency finishes.
try group.await(io);
}
@ -779,16 +790,17 @@ const Run = struct {
var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() });
defer cleanup_task.await(io);
for (step_stack.keys()) |s| {
test_pass_count += s.test_results.passCount();
test_skip_count += s.test_results.skip_count;
test_fail_count += s.test_results.fail_count;
test_crash_count += s.test_results.crash_count;
test_timeout_count += s.test_results.timeout_count;
for (step_stack.keys()) |step_index| {
const make_step = run.stepByIndex(step_index);
test_pass_count += make_step.test_results.passCount();
test_skip_count += make_step.test_results.skip_count;
test_fail_count += make_step.test_results.fail_count;
test_crash_count += make_step.test_results.crash_count;
test_timeout_count += make_step.test_results.timeout_count;
test_count += s.test_results.test_count;
test_count += make_step.test_results.test_count;
switch (s.state) {
switch (make_step.state) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
@ -797,7 +809,7 @@ const Run = struct {
.skipped, .skipped_oom => skipped_count += 1,
.failure => {
failure_count += 1;
const compile_errors_len = s.result_error_bundle.errorMessageCount();
const compile_errors_len = make_step.result_error_bundle.errorMessageCount();
if (compile_errors_len > 0) {
total_compile_errors += compile_errors_len;
}
@ -918,13 +930,17 @@ const Run = struct {
var print_node: PrintNode = .{ .parent = null };
if (step_names.len == 0) {
print_node.last = true;
printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {};
printTreeStep(run, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) {
error.Canceled => |e| return e,
else => {},
};
} else {
const last_index = if (run.summary == .all) top_level_steps.count() else blk: {
var i: usize = step_names.len;
while (i > 0) {
i -= 1;
const step = top_level_steps.get(step_names[i]).?.step;
const step_index = top_level_steps.get(step_names[i]).?;
const step = run.stepByIndex(step_index);
const found = switch (run.summary) {
.all, .line, .none => unreachable,
.failures => step.state != .success,
@ -935,9 +951,12 @@ const Run = struct {
break :blk top_level_steps.count();
};
for (step_names, 0..) |step_name, i| {
const tls = top_level_steps.get(step_name).?;
const step_index = top_level_steps.get(step_name).?;
print_node.last = i + 1 == last_index;
printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {};
printTreeStep(run, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) {
error.Canceled => |e| return e,
else => {},
};
}
}
w.writeByte('\n') catch {};
@ -957,120 +976,331 @@ const Run = struct {
_ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
process.exit(code);
}
};
const PrintNode = struct {
parent: ?*PrintNode,
last: bool = false,
};
fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {
const parent = node.parent orelse return;
const writer = stderr.writer;
if (parent.parent == null) return;
try printPrefix(parent, stderr);
if (parent.last) {
try writer.writeAll(" ");
} else {
try writer.writeAll(switch (stderr.mode) {
.escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │
else => "| ",
});
fn stepReady(
run: *Run,
group: *Io.Group,
step_index: Configuration.Step.Index,
root_prog_node: std.Progress.Node,
) Io.Cancelable!void {
const graph = run.graph;
const io = graph.io;
const c = &run.scanned_config.configuration;
const max_rss = step_index.ptr(c).max_rss.toBytes();
if (max_rss != 0) {
try run.max_rss_mutex.lock(io);
defer run.max_rss_mutex.unlock(io);
if (run.available_rss < max_rss) {
// Running this step right now could possibly exceed the allotted RSS.
run.memory_blocked_steps.append(run.gpa, step_index) catch
@panic("TODO eliminate memory allocation here");
return;
}
run.available_rss -= max_rss;
}
group.async(io, makeStep, .{ run, group, step_index, root_prog_node });
}
}
fn printChildNodePrefix(stderr: Io.Terminal) !void {
try stderr.writer.writeAll(switch (stderr.mode) {
.escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─
else => "+- ",
});
}
/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
/// have already subtracted this value from `run.available_rss`. This function will release the RSS
/// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked
/// steps after "make" completes for `s`.
fn makeStep(
run: *Run,
group: *Io.Group,
step_index: Configuration.Step.Index,
root_prog_node: std.Progress.Node,
) Io.Cancelable!void {
const graph = run.graph;
const io = graph.io;
const gpa = run.gpa;
const c = &run.scanned_config.configuration;
const conf_step = step_index.ptr(c);
const step_name = conf_step.name.slice(c);
const deps = conf_step.deps.slice(c);
const make_step = run.stepByIndex(step_index);
fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void {
const writer = stderr.writer;
switch (s.state) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
{
const step_prog_node = root_prog_node.start(step_name, 0);
defer step_prog_node.end();
.dependency_failure => {
try stderr.setColor(.dim);
try writer.writeAll(" transitive failure\n");
try stderr.setColor(.reset);
},
if (run.web_server) |*ws| ws.updateStepStatus(step_index, .wip);
.success => {
try stderr.setColor(.green);
if (s.result_cached) {
try writer.writeAll(" cached");
} else if (s.test_results.test_count > 0) {
const pass_count = s.test_results.passCount();
assert(s.test_results.test_count == pass_count + s.test_results.skip_count);
try writer.print(" {d} pass", .{pass_count});
if (s.test_results.skip_count > 0) {
try stderr.setColor(.reset);
try writer.writeAll(", ");
try stderr.setColor(.yellow);
try writer.print("{d} skip", .{s.test_results.skip_count});
const new_state: Step.State = for (deps) |dep_index| {
const dep_make_step = run.stepByIndex(dep_index);
switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
.failure,
.dependency_failure,
.skipped_oom,
=> break .dependency_failure,
.success, .skipped => {},
}
try stderr.setColor(.reset);
try writer.print(" ({d} total)", .{s.test_results.test_count});
} else if (make_step.make(.{
.progress_node = step_prog_node,
.watch = run.watch,
.web_server = if (run.web_server) |*ws| ws else null,
.unit_test_timeout_ns = run.unit_test_timeout_ns,
.gpa = gpa,
})) state: {
break :state .success;
} else |err| switch (err) {
error.MakeFailed => .failure,
error.MakeSkipped => .skipped,
};
@atomicStore(Step.State, &make_step.state, new_state, .monotonic);
switch (new_state) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
.failure,
.dependency_failure,
.skipped_oom,
=> {
if (run.web_server) |*ws| ws.updateStepStatus(step_index, .failure);
std.Progress.setStatus(.failure_working);
},
.success,
.skipped,
=> {
if (run.web_server) |*ws| ws.updateStepStatus(step_index, .success);
},
}
}
// No matter the result, we want to display error/warning messages.
if (make_step.result_error_bundle.errorMessageCount() > 0 or
make_step.result_error_msgs.items.len > 0 or
make_step.result_stderr.len > 0)
{
const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
defer io.unlockStderr();
printErrorMessages(gpa, c, run.steps, step_index, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch |err| switch (err) {
error.Canceled => |e| return e,
error.WriteFailed => switch (stderr.file_writer.err.?) {
error.Canceled => |e| return e,
else => {},
},
else => {},
};
}
const max_rss = conf_step.max_rss.toBytes();
if (max_rss != 0) {
var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty;
defer dispatch_set.deinit(gpa);
// Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
// as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held.
{
try run.max_rss_mutex.lock(io);
defer run.max_rss_mutex.unlock(io);
run.available_rss += max_rss;
dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch
@panic("TODO eliminate memory allocation here");
while (run.memory_blocked_steps.getLastOrNull()) |candidate_index| {
const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes();
if (run.available_rss < candidate_max_rss) break;
assert(run.memory_blocked_steps.pop() == candidate_index);
dispatch_set.appendAssumeCapacity(candidate_index);
}
}
for (dispatch_set.items) |candidate| {
group.async(io, makeStep, .{ run, group, candidate, root_prog_node });
}
}
for (make_step.dependants.items) |dependant_index| {
const dependant = run.stepByIndex(dependant_index);
// `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
try stepReady(run, group, dependant_index, root_prog_node);
}
}
}
fn printTreeStep(
run: *const Run,
step_index: Configuration.Step.Index,
stderr: Io.Terminal,
parent_node: *PrintNode,
step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
) !void {
const writer = stderr.writer;
const first = step_stack.swapRemove(step_index);
const summary = run.summary;
const c = &run.scanned_config.configuration;
const conf_step = step_index.ptr(c);
const make_step = run.stepByIndex(step_index);
const skip = switch (summary) {
.none, .line => unreachable,
.all => false,
.new => make_step.result_cached,
.failures => make_step.state == .success,
};
if (skip) return;
try printPrefix(parent_node, stderr);
if (parent_node.parent != null) {
if (parent_node.last) {
try printChildNodePrefix(stderr);
} else {
try writer.writeAll(" success");
try writer.writeAll(switch (stderr.mode) {
.escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─
else => "+- ",
});
}
try stderr.setColor(.reset);
if (s.result_duration_ns) |ns| {
try stderr.setColor(.dim);
if (ns >= std.time.ns_per_min) {
try writer.print(" {d}m", .{ns / std.time.ns_per_min});
} else if (ns >= std.time.ns_per_s) {
try writer.print(" {d}s", .{ns / std.time.ns_per_s});
} else if (ns >= std.time.ns_per_ms) {
try writer.print(" {d}ms", .{ns / std.time.ns_per_ms});
} else if (ns >= std.time.ns_per_us) {
try writer.print(" {d}us", .{ns / std.time.ns_per_us});
} else {
try writer.print(" {d}ns", .{ns});
}
try stderr.setColor(.reset);
}
if (s.result_peak_rss != 0) {
const rss = s.result_peak_rss;
try stderr.setColor(.dim);
if (rss >= 1000_000_000) {
try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000});
} else if (rss >= 1000_000) {
try writer.print(" MaxRSS:{d}M", .{rss / 1000_000});
} else if (rss >= 1000) {
try writer.print(" MaxRSS:{d}K", .{rss / 1000});
} else {
try writer.print(" MaxRSS:{d}B", .{rss});
}
try stderr.setColor(.reset);
}
try writer.writeAll("\n");
},
.skipped => {
try stderr.setColor(.yellow);
try writer.writeAll(" skipped\n");
try stderr.setColor(.reset);
},
.skipped_oom => {
try stderr.setColor(.yellow);
try writer.writeAll(" skipped (not enough memory)");
try stderr.setColor(.dim);
try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss });
try stderr.setColor(.reset);
},
.failure => {
try printStepFailure(s, stderr, false);
try stderr.setColor(.reset);
},
}
}
}
fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void {
if (!first) try stderr.setColor(.dim);
// dep_prefix omitted here because it is redundant with the tree.
try writer.writeAll(conf_step.name.slice(c));
const deps = conf_step.deps.slice(c);
if (first) {
try printStepStatus(run, step_index, stderr);
const last_index = if (summary == .all) deps.len -| 1 else blk: {
var i: usize = deps.len;
while (i > 0) {
i -= 1;
const dep_index = deps[i];
const dep = run.stepByIndex(dep_index);
const found = switch (summary) {
.all, .line, .none => unreachable,
.failures => dep.state != .success,
.new => !dep.result_cached,
};
if (found) break :blk i;
}
break :blk deps.len -| 1;
};
for (deps, 0..) |dep, i| {
var print_node: PrintNode = .{
.parent = parent_node,
.last = i == last_index,
};
try printTreeStep(run, dep, stderr, &print_node, step_stack);
}
} else {
if (deps.len == 0) {
try writer.writeAll(" (reused)\n");
} else {
try writer.print(" (+{d} more reused dependencies)\n", .{deps.len});
}
try stderr.setColor(.reset);
}
}
fn printStepStatus(run: *const Run, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void {
const s = run.stepByIndex(step_index);
const writer = stderr.writer;
switch (s.state) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
.dependency_failure => {
try stderr.setColor(.dim);
try writer.writeAll(" transitive failure\n");
try stderr.setColor(.reset);
},
.success => {
try stderr.setColor(.green);
if (s.result_cached) {
try writer.writeAll(" cached");
} else if (s.test_results.test_count > 0) {
const pass_count = s.test_results.passCount();
assert(s.test_results.test_count == pass_count + s.test_results.skip_count);
try writer.print(" {d} pass", .{pass_count});
if (s.test_results.skip_count > 0) {
try stderr.setColor(.reset);
try writer.writeAll(", ");
try stderr.setColor(.yellow);
try writer.print("{d} skip", .{s.test_results.skip_count});
}
try stderr.setColor(.reset);
try writer.print(" ({d} total)", .{s.test_results.test_count});
} else {
try writer.writeAll(" success");
}
try stderr.setColor(.reset);
if (s.result_duration_ns) |ns| {
try stderr.setColor(.dim);
if (ns >= std.time.ns_per_min) {
try writer.print(" {d}m", .{ns / std.time.ns_per_min});
} else if (ns >= std.time.ns_per_s) {
try writer.print(" {d}s", .{ns / std.time.ns_per_s});
} else if (ns >= std.time.ns_per_ms) {
try writer.print(" {d}ms", .{ns / std.time.ns_per_ms});
} else if (ns >= std.time.ns_per_us) {
try writer.print(" {d}us", .{ns / std.time.ns_per_us});
} else {
try writer.print(" {d}ns", .{ns});
}
try stderr.setColor(.reset);
}
if (s.result_peak_rss != 0) {
const rss = s.result_peak_rss;
try stderr.setColor(.dim);
if (rss >= 1000_000_000) {
try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000});
} else if (rss >= 1000_000) {
try writer.print(" MaxRSS:{d}M", .{rss / 1000_000});
} else if (rss >= 1000) {
try writer.print(" MaxRSS:{d}K", .{rss / 1000});
} else {
try writer.print(" MaxRSS:{d}B", .{rss});
}
try stderr.setColor(.reset);
}
try writer.writeAll("\n");
},
.skipped => {
try stderr.setColor(.yellow);
try writer.writeAll(" skipped\n");
try stderr.setColor(.reset);
},
.skipped_oom => {
const c = &run.scanned_config.configuration;
const max_rss = step_index.ptr(c).max_rss.toBytes();
try stderr.setColor(.yellow);
try writer.writeAll(" skipped (not enough memory)");
try stderr.setColor(.dim);
try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{
max_rss, run.available_rss,
});
try stderr.setColor(.reset);
},
.failure => {
try printStepFailure(run.steps, step_index, stderr, false);
try stderr.setColor(.reset);
},
}
}
};
fn printStepFailure(
make_steps: []Step,
step_index: Configuration.Step.Index,
stderr: Io.Terminal,
dim: bool,
) !void {
const w = stderr.writer;
const s = &make_steps[@intFromEnum(step_index)];
if (s.result_error_bundle.errorMessageCount() > 0) {
try stderr.setColor(.red);
try w.print(" {d} errors\n", .{
@ -1153,79 +1383,33 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void {
}
}
fn printTreeStep(
graph: *Graph,
s: *Step,
run: *const Run,
stderr: Io.Terminal,
parent_node: *PrintNode,
step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),
) !void {
const PrintNode = struct {
parent: ?*PrintNode,
last: bool = false,
};
fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {
const parent = node.parent orelse return;
const writer = stderr.writer;
const first = step_stack.swapRemove(s);
const summary = run.summary;
const skip = switch (summary) {
.none, .line => unreachable,
.all => false,
.new => s.result_cached,
.failures => s.state == .success,
};
if (skip) return;
try printPrefix(parent_node, stderr);
if (parent_node.parent != null) {
if (parent_node.last) {
try printChildNodePrefix(stderr);
} else {
try writer.writeAll(switch (stderr.mode) {
.escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─
else => "+- ",
});
}
}
if (!first) try stderr.setColor(.dim);
// dep_prefix omitted here because it is redundant with the tree.
try writer.writeAll(s.name);
if (first) {
try printStepStatus(s, stderr, run);
const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: {
var i: usize = s.dependencies.items.len;
while (i > 0) {
i -= 1;
const step = s.dependencies.items[i];
const found = switch (summary) {
.all, .line, .none => unreachable,
.failures => step.state != .success,
.new => !step.result_cached,
};
if (found) break :blk i;
}
break :blk s.dependencies.items.len -| 1;
};
for (s.dependencies.items, 0..) |dep, i| {
var print_node: PrintNode = .{
.parent = parent_node,
.last = i == last_index,
};
try printTreeStep(graph, dep, run, stderr, &print_node, step_stack);
}
if (parent.parent == null) return;
try printPrefix(parent, stderr);
if (parent.last) {
try writer.writeAll(" ");
} else {
if (s.dependencies.items.len == 0) {
try writer.writeAll(" (reused)\n");
} else {
try writer.print(" (+{d} more reused dependencies)\n", .{
s.dependencies.items.len,
});
}
try stderr.setColor(.reset);
try writer.writeAll(switch (stderr.mode) {
.escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │
else => "| ",
});
}
}
fn printChildNodePrefix(stderr: Io.Terminal) !void {
try stderr.writer.writeAll(switch (stderr.mode) {
.escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─
else => "+- ",
});
}
/// Traverse the dependency graph depth-first and make it undirected by having
/// steps know their dependants (they only know dependencies at start).
/// Along the way, check that there is no dependency loop, and record the steps
@ -1245,14 +1429,14 @@ fn constructGraphAndCheckForDependencyLoop(
step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
rand: std.Random,
) error{ DependencyLoopDetected, OutOfMemory }!void {
const s: *Step = &steps[@intFromEnum(step_index)];
switch (s.state) {
const make_step: *Step = &steps[@intFromEnum(step_index)];
switch (make_step.state) {
.precheck_started => {
log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)});
return error.DependencyLoopDetected;
},
.precheck_unstarted => {
s.state = .precheck_started;
make_step.state = .precheck_started;
const step = step_index.ptr(c);
const dependencies = step.deps.slice(c);
@ -1268,7 +1452,7 @@ fn constructGraphAndCheckForDependencyLoop(
for (deps) |dep| {
const dep_step: *Step = &steps[@intFromEnum(dep)];
try step_stack.put(gpa, dep, {});
try dep_step.dependants.append(gpa, s);
try dep_step.dependants.append(gpa, step_index);
constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) {
error.DependencyLoopDetected => {
log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)});
@ -1278,8 +1462,8 @@ fn constructGraphAndCheckForDependencyLoop(
};
}
s.state = .precheck_done;
s.pending_deps = @intCast(dependencies.len);
make_step.state = .precheck_done;
make_step.pending_deps = @intCast(dependencies.len);
},
.precheck_done => {},
@ -1292,140 +1476,11 @@ fn constructGraphAndCheckForDependencyLoop(
}
}
/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
/// have already subtracted this value from `run.available_rss`. This function will release the RSS
/// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked
/// steps after "make" completes for `s`.
fn makeStep(
graph: *Graph,
group: *Io.Group,
s: *Step,
root_prog_node: std.Progress.Node,
run: *Run,
) Io.Cancelable!void {
const io = graph.io;
const gpa = run.gpa;
{
const step_prog_node = root_prog_node.start(s.name, 0);
defer step_prog_node.end();
if (run.web_server) |*ws| ws.updateStepStatus(s, .wip);
const new_state: Step.State = for (s.dependencies.items) |dep| {
switch (@atomicLoad(Step.State, &dep.state, .monotonic)) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
.failure,
.dependency_failure,
.skipped_oom,
=> break .dependency_failure,
.success, .skipped => {},
}
} else if (s.make(.{
.progress_node = step_prog_node,
.watch = run.watch,
.web_server = if (run.web_server) |*ws| ws else null,
.unit_test_timeout_ns = run.unit_test_timeout_ns,
.gpa = gpa,
})) state: {
break :state .success;
} else |err| switch (err) {
error.MakeFailed => .failure,
error.MakeSkipped => .skipped,
};
@atomicStore(Step.State, &s.state, new_state, .monotonic);
switch (new_state) {
.precheck_unstarted => unreachable,
.precheck_started => unreachable,
.precheck_done => unreachable,
.failure,
.dependency_failure,
.skipped_oom,
=> {
if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);
std.Progress.setStatus(.failure_working);
},
.success,
.skipped,
=> {
if (run.web_server) |*ws| ws.updateStepStatus(s, .success);
},
}
}
// No matter the result, we want to display error/warning messages.
if (s.result_error_bundle.errorMessageCount() > 0 or
s.result_error_msgs.items.len > 0 or
s.result_stderr.len > 0)
{
const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
defer io.unlockStderr();
printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {};
}
if (s.max_rss != 0) {
var dispatch_set: std.ArrayList(*Step) = .empty;
defer dispatch_set.deinit(gpa);
// Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
// as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held.
{
try run.max_rss_mutex.lock(io);
defer run.max_rss_mutex.unlock(io);
run.available_rss += s.max_rss;
try dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len);
while (run.memory_blocked_steps.getLastOrNull()) |candidate| {
if (run.available_rss < candidate.max_rss) break;
assert(run.memory_blocked_steps.pop() == candidate);
dispatch_set.appendAssumeCapacity(candidate);
}
}
for (dispatch_set.items) |candidate| {
group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run });
}
}
for (s.dependants.items) |dependant| {
// `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
try stepReady(graph, group, dependant, root_prog_node, run);
}
}
}
fn stepReady(
graph: *Graph,
group: *Io.Group,
s: *Step,
root_prog_node: std.Progress.Node,
run: *Run,
) !void {
const io = graph.io;
if (s.max_rss != 0) {
try run.max_rss_mutex.lock(io);
defer run.max_rss_mutex.unlock(io);
if (run.available_rss < s.max_rss) {
// Running this step right now could possibly exceed the allotted RSS.
try run.memory_blocked_steps.append(run.gpa, s);
return;
}
run.available_rss -= s.max_rss;
}
group.async(io, makeStep, .{ graph, group, s, root_prog_node, run });
}
pub fn printErrorMessages(
gpa: Allocator,
failing_step: *Step,
c: *const Configuration,
make_steps: []Step,
failing_step_index: Configuration.Step.Index,
options: std.zig.ErrorBundle.RenderOptions,
stderr: Io.Terminal,
error_style: ErrorStyle,
@ -1435,26 +1490,28 @@ pub fn printErrorMessages(
if (error_style.verboseContext()) {
// Provide context for where these error messages are coming from by
// printing the corresponding Step subtree.
var step_stack: std.ArrayList(*Step) = .empty;
var step_stack: std.ArrayList(Configuration.Step.Index) = .empty;
defer step_stack.deinit(gpa);
try step_stack.append(gpa, failing_step);
while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) {
try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]);
try step_stack.append(gpa, failing_step_index);
while (true) {
const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])];
if (last_step.dependants.items.len == 0) break;
try step_stack.append(gpa, last_step.dependants.items[0]);
}
// Now, `step_stack` has the subtree that we want to print, in reverse order.
try stderr.setColor(.dim);
var indent: usize = 0;
while (step_stack.pop()) |s| : (indent += 1) {
while (step_stack.pop()) |step_index| : (indent += 1) {
if (indent > 0) {
try writer.splatByteAll(' ', (indent - 1) * 3);
try printChildNodePrefix(stderr);
}
try writer.writeAll(s.name);
try writer.writeAll(step_index.ptr(c).name.slice(c));
if (s == failing_step) {
try printStepFailure(s, stderr, true);
if (step_index == failing_step_index) {
try printStepFailure(make_steps, step_index, stderr, true);
} else {
try writer.writeAll("\n");
}
@ -1463,11 +1520,13 @@ pub fn printErrorMessages(
} else {
// Just print the failing step itself.
try stderr.setColor(.dim);
try writer.writeAll(failing_step.name);
try printStepFailure(failing_step, stderr, true);
try writer.writeAll(failing_step_index.ptr(c).name.slice(c));
try printStepFailure(make_steps, failing_step_index, stderr, true);
try stderr.setColor(.reset);
}
const failing_step = &make_steps[@intFromEnum(failing_step_index)];
if (failing_step.result_stderr.len > 0) {
try writer.writeAll(failing_step.result_stderr);
if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) {
@ -1560,9 +1619,10 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {
fatal(f, args);
}
fn cleanTmpFiles(io: Io, steps: []const *Step) void {
for (steps) |step| {
const wf = step.cast(Step.WriteFile) orelse continue;
fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void {
for (steps) |step_index| {
if (true) @panic("TODO");
const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue;
if (wf.mode != .tmp) continue;
const path = wf.generated_directory.path orelse continue;
Io.Dir.cwd().deleteTree(io, path) catch |err| {

54
lib/compiler/maker/Fuzz.zig
View file

@ -1,16 +1,16 @@
const Fuzz = @This();
const std = @import("std");
const Io = std.Io;
const Allocator = std.mem.Allocator;
const Build = std.Build;
const Cache = std.Build.Cache;
const Step = std.Build.Step;
const Coverage = std.debug.Coverage;
const Configuration = std.Build.Configuration;
const Io = std.Io;
const abi = std.Build.abi.fuzz;
const assert = std.debug.assert;
const fatal = std.process.fatal;
const Allocator = std.mem.Allocator;
const log = std.log;
const Coverage = std.debug.Coverage;
const abi = std.Build.abi.fuzz;
const maker = @import("../maker.zig");
const WebServer = @import("WebServer.zig");
@ -20,7 +20,7 @@ io: Io,
mode: Mode,
/// Allocated into `gpa`.
run_steps: []const *Step.Run,
run_steps: []const Configuration.Step.Index,
group: Io.Group,
root_prog_node: std.Progress.Node,
@ -51,7 +51,7 @@ const Msg = union(enum) {
unique: u64,
coverage: u64,
},
run: *Step.Run,
run: Configuration.Step.Index,
},
entry_point: struct {
coverage_id: u64,
@ -78,12 +78,12 @@ const CoverageMap = struct {
pub fn init(
gpa: Allocator,
io: Io,
all_steps: []const *Build.Step,
all_steps: []const Configuration.Step.Index,
root_prog_node: std.Progress.Node,
mode: Mode,
) error{ OutOfMemory, Canceled }!Fuzz {
const run_steps: []const *Step.Run = steps: {
var steps: std.ArrayList(*Step.Run) = .empty;
const run_steps: []const Configuration.Step.Index = steps: {
var steps: std.ArrayList(Configuration.Step.Index) = .empty;
defer steps.deinit(gpa);
const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
defer rebuild_node.end();
@ -91,7 +91,8 @@ pub fn init(
defer rebuild_group.cancel(io);
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
if (true) @panic("TODO");
const run = step.cast(std.Build.Step.Run) orelse continue;
if (run.producer == null) continue;
if (run.fuzz_tests.items.len == 0) continue;
try steps.append(gpa, run);
@ -100,15 +101,16 @@ pub fn init(
if (steps.items.len == 0) fatal("no fuzz tests found", .{});
rebuild_node.setEstimatedTotalItems(steps.items.len);
const run_steps = try gpa.dupe(*Step.Run, steps.items);
const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items);
try rebuild_group.await(io);
break :steps run_steps;
};
errdefer gpa.free(run_steps);
for (run_steps) |run| {
assert(run.fuzz_tests.items.len > 0);
if (run.rebuilt_executable == null)
for (run_steps) |run_step_index| {
if (true) @panic("TODO");
assert(run_step_index.fuzz_tests.items.len > 0);
if (run_step_index.rebuilt_executable == null)
fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
}
@ -147,6 +149,7 @@ pub fn start(fuzz: *Fuzz) void {
}
for (fuzz.run_steps) |run| {
if (true) @panic("TODO");
for (run.fuzz_tests.items) |unit_test_index| {
assert(run.rebuilt_executable != null);
fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });
@ -161,14 +164,14 @@ pub fn deinit(fuzz: *Fuzz) void {
fuzz.gpa.free(fuzz.run_steps);
}
fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) void {
fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void {
rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| {
const compile = run.producer.?;
log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });
};
}
fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) !void {
fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void {
const graph = run.step.owner.graph;
const io = graph.io;
const compile = run.producer.?;
@ -195,7 +198,7 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod
run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
}
fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index, unit_test_index: u32) void {
const owner = run.step.owner;
const gpa = owner.allocator;
const graph = owner.graph;
@ -223,6 +226,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
}
pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
if (true) @panic("TODO");
assert(fuzz.mode == .forever);
var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
@ -368,7 +372,8 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {
fuzz.msg_queue.clearRetainingCapacity();
}
}
fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
if (true) @panic("TODO");
assert(fuzz.mode == .forever);
const ws = fuzz.mode.forever.ws;
const gpa = fuzz.gpa;
@ -398,8 +403,8 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
};
errdefer gop.value_ptr.coverage.deinit(gpa);
const rebuilt_exe_path = run_step.rebuilt_executable.?;
const target = run_step.producer.?.rootModuleTarget();
const rebuilt_exe_path = run_step_index.rebuilt_executable.?;
const target = run_step_index.producer.?.rootModuleTarget();
var debug_info = std.debug.Info.load(
gpa,
io,
@ -409,19 +414,19 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
target.cpu.arch,
) catch |err| {
log.err("step '{s}': failed to load debug information for '{f}': {t}", .{
run_step.step.name, rebuilt_exe_path, err,
run_step_index.step.name, rebuilt_exe_path, err,
});
return error.AlreadyReported;
};
defer debug_info.deinit(gpa);
const coverage_file_path: Build.Cache.Path = .{
.root_dir = run_step.step.owner.cache_root,
.root_dir = run_step_index.step.owner.cache_root,
.sub_path = "v/" ++ std.fmt.hex(coverage_id),
};
var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {
log.err("step '{s}': failed to load coverage file '{f}': {t}", .{
run_step.step.name, coverage_file_path, err,
run_step_index.step.name, coverage_file_path, err,
});
return error.AlreadyReported;
};
@ -528,6 +533,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte
}
pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
if (true) @panic("TODO");
assert(fuzz.mode == .limit);
const io = fuzz.io;

6
lib/compiler/maker/Step.zig
View file

@ -10,6 +10,7 @@ const Io = std.Io;
const LazyPath = std.Build.Configuration.LazyPath;
const Package = std.Build.Configuration.Package;
const Path = std.Build.Cache.Path;
const Configuration = std.Build.Configuration;
const assert = std.debug.assert;
const WebServer = @import("WebServer.zig");
@ -21,7 +22,7 @@ pub const Run = void; // @import("Step/Run.zig");
_: void align(std.atomic.cache_line) = {},
state: State = .precheck_unstarted,
dependants: std.ArrayList(*Step) = .empty,
dependants: std.ArrayList(Configuration.Step.Index) = .empty,
/// Collects the set of files that retrigger this step to run.
///
/// This is used by the build system's implementation of `--watch` but it can
@ -143,6 +144,7 @@ pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
/// have already reported the error. Otherwise, we add a simple error report
/// here.
pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
if (true) @panic("TODO Step.make");
const arena = s.owner.allocator;
const graph = s.owner.graph;
const io = graph.io;
@ -183,6 +185,7 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi
/// Implementation detail of file watching. Prepares the step for being re-evaluated.
/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated.
pub fn invalidateResult(step: *Step, gpa: Allocator) bool {
if (true) @panic("TODO Step.invalidateResult");
if (step.state == .precheck_done) return false;
assert(step.pending_deps == 0);
step.state = .precheck_done;
@ -544,6 +547,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer
}
pub fn getZigProcess(s: *Step) ?*ZigProcess {
if (true) @panic("TODO getZigProcess");
return switch (s.id) {
.compile => s.cast(Compile).?.zig_process,
else => null,

52
lib/compiler/maker/Watch.zig
View file

@ -3,11 +3,13 @@ const builtin = @import("builtin");
const std = @import("std");
const Io = std.Io;
const Step = std.Build.Step;
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const fatal = std.process.fatal;
const Configuration = std.Build.Configuration;
const FsEvents = @import("Watch/FsEvents.zig");
const Step = @import("Step.zig");
os: Os,
/// The number to show as the number of directories being watched.
@ -16,6 +18,8 @@ dir_count: usize,
// They are `undefined` on implementations which do not utilize then.
dir_table: DirTable,
generation: Generation,
configuration: *const Configuration,
make_steps: []Step,
pub const have_impl = Os != void;
@ -27,7 +31,7 @@ const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAda
/// Special key of "." means any changes in this directory trigger the steps.
const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet);
const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation);
const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation);
const Generation = u8;
@ -101,7 +105,7 @@ const Os = switch (builtin.os.tag) {
};
};
fn init(cwd_path: []const u8) !Watch {
fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
_ = cwd_path;
return .{
.dir_table = .{},
@ -114,6 +118,8 @@ const Os = switch (builtin.os.tag) {
else => {},
},
.generation = 0,
.make_steps = make_steps,
.configuration = configuration,
};
}
@ -161,20 +167,21 @@ const Os = switch (builtin.os.tag) {
const lfh: FileHandle = .{ .handle = file_handle };
if (w.os.handle_table.getPtr(lfh)) |value| {
if (value.reaction_set.getPtr(".")) |glob_set|
any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty);
if (value.reaction_set.getPtr(file_name)) |step_set|
any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty);
}
},
else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}),
else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}),
}
}
}
}
fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
// Add missing marks and note persisted ones.
for (steps) |step| {
for (steps) |step_index| {
const step = &w.make_steps[@intFromEnum(step_index)];
for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
const reaction_set = rs: {
const gop = try w.dir_table.getOrPut(gpa, path);
@ -236,7 +243,7 @@ const Os = switch (builtin.os.tag) {
for (files.items) |basename| {
const gop = try reaction_set.getOrPut(gpa, basename);
if (!gop.found_existing) gop.value_ptr.* = .{};
try gop.value_ptr.put(gpa, step, w.generation);
try gop.value_ptr.put(gpa, step_index, w.generation);
}
}
}
@ -537,7 +544,7 @@ const Os = switch (builtin.os.tag) {
return any_dirty;
}
fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
// Add missing marks and note persisted ones.
for (steps) |step| {
for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
@ -678,7 +685,7 @@ const Os = switch (builtin.os.tag) {
};
}
fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
const handles = &w.os.handles;
for (steps) |step| {
for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
@ -856,7 +863,7 @@ const Os = switch (builtin.os.tag) {
.generation = undefined,
};
}
fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
try w.os.fse.setPaths(gpa, steps);
w.dir_count = w.os.fse.watch_roots.len;
}
@ -871,8 +878,8 @@ const Os = switch (builtin.os.tag) {
else => void,
};
pub fn init(cwd_path: []const u8) !Watch {
return Os.init(cwd_path);
pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
return Os.init(cwd_path, configuration, make_steps);
}
pub const Match = struct {
@ -880,20 +887,19 @@ pub const Match = struct {
/// match.
basename: []const u8,
/// The step to re-run when file corresponding to `basename` is changed.
step: *Step,
step_index: Configuration.Step.Index,
pub const Context = struct {
pub fn hash(self: Context, a: Match) u32 {
_ = self;
var hasher = Hash.init(0);
std.hash.autoHash(&hasher, a.step);
var hasher = Hash.init(@intFromEnum(a.step_index));
hasher.update(a.basename);
return @truncate(hasher.final());
}
pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
_ = self;
_ = b_index;
return a.step == b.step and std.mem.eql(u8, a.basename, b.basename);
return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename);
}
};
};
@ -908,22 +914,24 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
else => item,
};
for (reaction_set.values()) |step_set| {
for (step_set.keys()) |step| {
for (step_set.keys()) |step_index| {
const step = &w.make_steps[@intFromEnum(step_index)];
_ = step.invalidateResult(gpa);
}
}
}
}
fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {
fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool {
var this_any_dirty = false;
for (step_set.keys()) |step| {
for (step_set.keys()) |step_index| {
const step = &make_steps[@intFromEnum(step_index)];
if (step.invalidateResult(gpa)) this_any_dirty = true;
}
return any_dirty or this_any_dirty;
}
pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
return Os.update(w, gpa, steps);
}

61
lib/compiler/maker/WebServer.zig
View file

@ -4,8 +4,8 @@ const builtin = @import("builtin");
const std = @import("std");
const Allocator = std.mem.Allocator;
const Build = std.Build;
const Cache = std.Build.Cache;
const Configuration = std.Build.Configuration;
const Io = std.Io;
const abi = std.Build.abi;
const assert = std.debug.assert;
@ -15,10 +15,12 @@ const mem = std.mem;
const net = std.Io.net;
const Fuzz = @import("Fuzz.zig");
const Graph = @import("Graph.zig");
const Step = @import("Step.zig");
gpa: Allocator,
graph: *const Build.Graph,
all_steps: []const *Build.Step,
graph: *const Graph,
all_steps: []const Configuration.Step.Index,
listen_address: net.IpAddress,
root_prog_node: std.Progress.Node,
watch: bool,
@ -69,12 +71,13 @@ pub fn notifyUpdate(ws: *WebServer) void {
pub const Options = struct {
gpa: Allocator,
graph: *const std.Build.Graph,
all_steps: []const *Build.Step,
graph: *const Graph,
all_steps: []const Configuration.Step.Index,
root_prog_node: std.Progress.Node,
watch: bool,
listen_address: net.IpAddress,
base_timestamp: Io.Clock.Timestamp,
configuration: *const Configuration,
};
pub fn init(opts: Options) WebServer {
// The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent`
@ -83,19 +86,21 @@ pub fn init(opts: Options) WebServer {
assert(opts.base_timestamp.clock == base_clock);
const all_steps = opts.all_steps;
const c = opts.configuration;
const step_names_trailing = opts.gpa.alloc(u8, len: {
var name_bytes: usize = 0;
for (all_steps) |step| name_bytes += step.name.len;
for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len;
break :len name_bytes + all_steps.len * 4;
}) catch @panic("out of memory");
{
const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]);
var idx: usize = all_steps.len * 4;
for (all_steps, step_name_lens) |step, *name_len| {
name_len.* = @intCast(step.name.len);
@memcpy(step_names_trailing[idx..][0..step.name.len], step.name);
idx += step.name.len;
for (all_steps, step_name_lens) |step_index, *name_len| {
const step_name = step_index.ptr(c).name.slice(c);
name_len.* = @intCast(step_name.len);
@memcpy(step_names_trailing[idx..][0..step_name.len], step_name);
idx += step_name.len;
}
assert(idx == step_names_trailing.len);
}
@ -208,9 +213,14 @@ pub fn startBuild(ws: *WebServer) void {
ws.notifyUpdate();
}
pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void {
pub fn updateStepStatus(
ws: *WebServer,
step_index: Configuration.Step.Index,
new_status: abi.StepUpdate.Status,
) void {
// TODO don't do linear search, especially in a hot loop like this
const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
if (s == step) break @intCast(i);
if (s == step_index) break @intCast(i);
} else unreachable;
const ptr = &ws.step_status_bits[step_idx / 4];
const bit_offset: u3 = @intCast((step_idx % 4) * 2);
@ -673,7 +683,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
if (code != 0) {
log.err(
"the following command exited with error code {d}:\n{s}",
.{ code, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) },
.{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
);
return error.WasmCompilationFailed;
}
@ -681,14 +691,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
.signal => |sig| {
log.err(
"the following command terminated with signal {t}:\n{s}",
.{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) },
.{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
);
return error.WasmCompilationFailed;
},
.stopped, .unknown => {
log.err(
"the following command terminated unexpectedly:\n{s}",
.{try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)},
.{try Step.allocPrintCmd(arena, .inherit, null, argv.items)},
);
return error.WasmCompilationFailed;
},
@ -698,14 +708,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
try result_error_bundle.renderToStderr(io, .{}, .auto);
log.err("the following command failed with {d} compilation errors:\n{s}", .{
result_error_bundle.errorMessageCount(),
try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items),
try Step.allocPrintCmd(arena, .inherit, null, argv.items),
});
return error.WasmCompilationFailed;
}
const base_path = result orelse {
log.err("child process failed to report result\n{s}", .{
try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items),
try Step.allocPrintCmd(arena, .inherit, null, argv.items),
});
return error.WasmCompilationFailed;
};
@ -729,7 +739,7 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8
}
pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
compile: *Build.Step.Compile,
compile_step: Configuration.Step.Index,
use_llvm: bool,
stats: abi.time_report.CompileResult.Stats,
@ -745,8 +755,9 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
const gpa = ws.gpa;
const io = ws.graph.io;
// TODO don't do linear search
const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
if (s == &opts.compile.step) break @intCast(i);
if (s == opts.compile_step) break @intCast(i);
} else unreachable;
const old_buf = old: {
@ -782,12 +793,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
ws.notifyUpdate();
}
pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.Duration) void {
pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void {
const gpa = ws.gpa;
const io = ws.graph.io;
// TODO don't do linear search
const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
if (s == step) break @intCast(i);
if (s == step_index) break @intCast(i);
} else unreachable;
const old_buf = old: {
@ -815,15 +827,16 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.D
pub fn updateTimeReportRunTest(
ws: *WebServer,
run: *Build.Step.Run,
tests: *const Build.Step.Run.CachedTestMetadata,
run_step_index: Configuration.Step.Index,
tests: *const Step.Run.CachedTestMetadata,
ns_per_test: []const u64,
) void {
const gpa = ws.gpa;
const io = ws.graph.io;
// TODO don't do linear search
const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
if (s == &run.step) break @intCast(i);
if (s == run_step_index) break @intCast(i);
} else unreachable;
assert(tests.names.len == ns_per_test.len);