Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/04-mcpp-toml.md
Original file line number Diff line number Diff line change
Expand Up @@ -937,6 +937,23 @@ with two behaviours has to be.

`bmi_schedule` decides when importers are unblocked.

`memory_threshold_percent` is a machine setting in `~/.mcpp/config.toml`, not a
project-manifest field. It defaults to `80` and limits the compiler pool using
the available memory observed at build start. `MCPP_MEMORY_THRESHOLD_PERCENT`
overrides it for one invocation. Ordinary C/C++ edges and both detached GCC and
two-phase Clang module edges use the same admission check. The admission baseline
is rounded up to a 4 GiB bucket. Ninja's jobs setting controls ordinary and
two-phase edge parallelism; detached GCC compilation retains its compiler count
cap. Under memory pressure, a shared single fallback slot still admits one
compiler. Unsupported memory metrics leave the configured job/compiler cap in
force.

```toml
# ~/.mcpp/config.toml
[build]
memory_threshold_percent = 80
```

| value | |
|---|---|
| `"auto"` | **the default, and it currently means OFF** |
Expand Down
14 changes: 14 additions & 0 deletions docs/zh/04-mcpp-toml.md
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,20 @@ bmi_schedule = "off" # auto (default, = off) | on | off

`bmi_schedule` 决定 importer 何时被解除阻塞。

`memory_threshold_percent` 是 `~/.mcpp/config.toml` 中的本机设置,不是
工程 manifest 字段。默认值为 `80`,它依据构建开始时观测到的可用内存限制
编译器池;单次调用可以用 `MCPP_MEMORY_THRESHOLD_PERCENT` 覆盖。普通
C/C++ 编译边、GCC 分离式模块边和 Clang 两阶段模块边使用同一条准入检查;
准入基线按 4 GiB 向上取整。普通边和两阶段边的并行度由 Ninja 的 jobs 设置
决定,GCC 分离式编译保留编译器数量上限。内存不足时,共享的单槽兜底仍可
放行一个编译。不受支持的内存指标不会取消已经配置的 jobs/编译器上限。

```toml
# ~/.mcpp/config.toml
[build]
memory_threshold_percent = 80
```

| 取值 | |
|---|---|
| `"auto"` | **默认值,目前意味着关闭** |
Expand Down
99 changes: 99 additions & 0 deletions modules/platform/src/capacity.cppm
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,32 @@ struct HostCapacity {
unsigned long long availableBytes = 0; // falls back to total when unknown
};

struct MemorySnapshot {
unsigned long long availableBytes = 0;
bool supported = false;
};

HostCapacity host_capacity();

// A fresh reading for admission control. Unlike host_capacity(), this is
// intentionally cheap and has no CPU topology work. `supported == false` means
// that callers must use their safe fallback rather than treating zero as free
// memory.
MemorySnapshot memory_snapshot();

// Round observed available memory up to a 4 GiB bucket (1 GiB if unknown).
unsigned long long memory_baseline(unsigned long long availableBytes);

// Whether a new compiler may be admitted. The baseline is the available
// physical memory observed when the build started. `thresholdPercent` is the
// proportion of that baseline which may be consumed. `hysteresisPercent` is
// the extra margin required before admitting work again after pressure.
bool memory_allows(const MemorySnapshot& now,
unsigned long long baselineBytes,
int thresholdPercent,
bool recovering = false,
int hysteresisPercent = 5);

// Job count for `auto`. See the module comment for why this is not `nproc`.
//
// cpu_budget = heterogeneous ? physicalCores : logicalCores
Expand Down Expand Up @@ -128,6 +152,11 @@ HostCapacity host_capacity() {
return cap;
}

MemorySnapshot memory_snapshot() {
const auto cap = host_capacity();
return {cap.availableBytes, cap.availableBytes != 0};
}

#elif defined(__APPLE__)

HostCapacity host_capacity() {
Expand Down Expand Up @@ -163,6 +192,11 @@ HostCapacity host_capacity() {
return cap;
}

MemorySnapshot memory_snapshot() {
const auto cap = host_capacity();
return {cap.availableBytes, cap.availableBytes != 0};
}

#else // Linux and other POSIX

static bool read_meminfo_kb(const char* key, unsigned long long& out) {
Expand All @@ -181,6 +215,35 @@ static bool read_meminfo_kb(const char* key, unsigned long long& out) {
return found;
}

static bool read_file_u64(const char* path, unsigned long long& out) {
FILE* f = ::fopen(path, "r");
if (!f) return false;
char value[128]{};
const bool ok = ::fgets(value, sizeof(value), f) != nullptr;
::fclose(f);
if (!ok) return false;
if (::strcmp(value, "max\n") == 0 || ::strcmp(value, "max") == 0) return false;
char* end = nullptr;
const auto n = ::strtoull(value, &end, 10);
if (end == value) return false;
out = n;
return true;
}

static void apply_cgroup_limit(unsigned long long& available) {
// cgroup v2 exposes the container's memory ceiling separately from the
// host's /proc/meminfo. A build must not use memory which the container
// cannot actually obtain. Keep this deliberately conservative: if the
// hierarchy cannot be discovered, the host reading remains valid.
unsigned long long max = 0, current = 0;
if (!read_file_u64("/sys/fs/cgroup/memory.max", max)
|| !read_file_u64("/sys/fs/cgroup/memory.current", current)
|| max == 0 || current >= max)
return;
const auto limited = max - current;
if (limited < available) available = limited;
}

static int read_cpuinfo_cores() {
FILE* f = ::fopen("/proc/cpuinfo", "r");
if (!f) return 0;
Expand Down Expand Up @@ -239,9 +302,17 @@ HostCapacity host_capacity() {
// page cache and would make every warm machine look starved.
if (!read_meminfo_kb("MemAvailable:", cap.availableBytes))
cap.availableBytes = cap.totalBytes;
apply_cgroup_limit(cap.availableBytes);
return cap;
}

MemorySnapshot memory_snapshot() {
unsigned long long available = 0;
bool supported = read_meminfo_kb("MemAvailable:", available);
if (supported) apply_cgroup_limit(available);
return {available, supported && available != 0};
}

#endif

int recommended_jobs(const HostCapacity& cap, unsigned long long perJobBytes,
Expand Down Expand Up @@ -269,4 +340,32 @@ int recommended_jobs(const HostCapacity& cap, unsigned long long perJobBytes,
return jobs;
}

unsigned long long memory_baseline(unsigned long long availableBytes) {
constexpr unsigned long long gib = 1024ULL * 1024ULL * 1024ULL;
constexpr unsigned long long bucket = 4 * gib;
if (availableBytes == 0) return gib;
const auto remainder = availableBytes % bucket;
if (remainder == 0) return availableBytes;
const auto increment = bucket - remainder;
// Saturate rather than wrap if an observation is near the integer limit.
constexpr auto maximum = ~0ULL;
return availableBytes > maximum - increment ? maximum : availableBytes + increment;
}

bool memory_allows(const MemorySnapshot& now, unsigned long long baselineBytes,
int thresholdPercent, bool recovering, int hysteresisPercent) {
if (!now.supported || baselineBytes == 0) return false;
if (thresholdPercent < 1) thresholdPercent = 1;
if (thresholdPercent > 100) thresholdPercent = 100;
if (hysteresisPercent < 0) hysteresisPercent = 0;
const int margin = recovering
? (thresholdPercent > hysteresisPercent
? thresholdPercent - hysteresisPercent : 1)
: thresholdPercent;
// `thresholdPercent` is the consumed portion. At 80%, at least 20% of the
// startup available memory must remain free.
const auto required = baselineBytes * static_cast<unsigned long long>(100 - margin) / 100;
return now.availableBytes >= required;
}

} // namespace mcpp::platform::capacity
18 changes: 18 additions & 0 deletions src/build/execute.cppm
Original file line number Diff line number Diff line change
Expand Up @@ -1367,6 +1367,22 @@ export bool dep_sources_newer_than(const std::vector<DepSourceRoot>& depSourceRo
// should abandon the fast path and fall back to a full prepare_build — or
// an exit code otherwise (0 success; 1 hard failure, diagnostics already
// printed to stderr).
std::optional<unsigned long long> read_ninja_memory_baseline(
const std::filesystem::path& ninjaPath) {
std::ifstream in(ninjaPath);
std::string line;
constexpr std::string_view prefix = "mcpp_mem_baseline = ";
while (std::getline(in, line)) {
if (!line.starts_with(prefix)) continue;
unsigned long long value = 0;
const auto text = std::string_view(line).substr(prefix.size());
const auto [end, ec] = std::from_chars(text.data(), text.data() + text.size(), value);
if (ec == std::errc{} && end == text.data() + text.size()) return value;
return std::nullopt;
}
return std::nullopt;
}

std::optional<int> run_ninja_fast(const std::string& ninjaProgram,
const std::filesystem::path& outputDir,
const std::filesystem::path& ninjaPath,
Expand Down Expand Up @@ -1417,6 +1433,8 @@ std::optional<int> run_ninja_fast(const std::string& ninjaProgram,
} else if (runtimeEnvKey != "-" && !runtimeEnvValue.empty()) {
childEnv.emplace_back(runtimeEnvKey, runtimeEnvValue);
}
if (auto baseline = read_ninja_memory_baseline(ninjaPath))
childEnv.emplace_back("MCPP_MEM_BASELINE", std::to_string(*baseline));

auto t0 = std::chrono::steady_clock::now();
// capture_exec merges stderr into the captured output (replacing `2>&1`),
Expand Down
Loading
Loading