From 374786f58a860b030cf6dd56604dc6e940c1a71c Mon Sep 17 00:00:00 2001 From: nella Date: Fri, 2 Oct 2026 16:26:38 +0200 Subject: [PATCH] Cache headers. --- src/compiler/lex/char_class.h | 12 +++++- src/compiler/lex/preprocess.cpp | 36 ++++++++++------- src/compiler/lex/preprocess_detail.h | 39 ++++++++++-------- src/compiler/lex/preprocess_lex.cpp | 57 +++++++++++++++++++++++++-- src/compiler/lex/preprocess_macro.cpp | 1 + 5 files changed, 108 insertions(+), 37 deletions(-) diff --git a/src/compiler/lex/char_class.h b/src/compiler/lex/char_class.h index e4d0e1b0..27018b33 100644 --- a/src/compiler/lex/char_class.h +++ b/src/compiler/lex/char_class.h @@ -4,10 +4,18 @@ #include "core.h" namespace rat::cc { - inline B32 isIdentStart(C8 c) { + constexpr B32 isIdentStart(C8 c) { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); } - inline B32 isIdentCont(C8 c) { return isIdentStart(c) || (c >= '0' && c <= '9'); } + struct IdentContTable { + U8 v[256] = {}; + constexpr IdentContTable() { + for(U32 c = 0; c < 256; ++c) + v[c] = isIdentStart((C8)c) || (c >= '0' && c <= '9'); + } + }; + inline constexpr IdentContTable kIdentCont; + inline B32 isIdentCont(C8 c) { return kIdentCont.v[(U8)c]; } inline B32 isDigit(C8 c) { return c >= '0' && c <= '9'; } inline B32 isOctalDigit(C8 c) { return c >= '0' && c <= '7'; } inline B32 isHexDigit(C8 c) { diff --git a/src/compiler/lex/preprocess.cpp b/src/compiler/lex/preprocess.cpp index 8c9a2154..05201b3a 100644 --- a/src/compiler/lex/preprocess.cpp +++ b/src/compiler/lex/preprocess.cpp @@ -113,7 +113,7 @@ namespace rat::cc { String found; String content; for(const String& p : tries) { - if(readFile(p, content)) { + if(fileToks.count(p) || readFile(p, content)) { found = p; break; } @@ -128,8 +128,11 @@ namespace rat::cc { fail("#include nesting too deep"); return; } + auto it = fileToks.find(found); + if(it == fileToks.end()) + it = fileToks.emplace(found, lexFragment(content, intern(found))).first; ++includeDepth; - runFile(found, content); + runTokens(found, it->second); --includeDepth; } @@ -229,7 +232,7 @@ namespace rat::cc { List Preprocessor::applyPragmaOperators(List& toks, const String& path) { B32 any = false; for(const PpToken& t : toks) - if(t.kind == Pk::Id && *t.text == "_Pragma") { + if(t.kind == Pk::Id && t.text == idPragma) { any = true; break; } @@ -237,7 +240,7 @@ namespace rat::cc { return std::move(toks); List out; for(U64 i = 0; i < toks.size();) { - if(toks[i].kind == Pk::Id && *toks[i].text == "_Pragma" && i + 3 < toks.size() && + if(toks[i].kind == Pk::Id && toks[i].text == idPragma && i + 3 < toks.size() && isPunct(toks[i + 1], "(") && toks[i + 2].kind == Pk::Str && isPunct(toks[i + 3], ")")) { List body = lexFragment(destringize(*toks[i + 2].text), intern(path)); doPragma(PpSpan(body), path); @@ -250,10 +253,10 @@ namespace rat::cc { return out; } - void Preprocessor::flush(List& textBuf) { - if(textBuf.empty()) + void Preprocessor::flush(PpSpan text) { + if(text.empty()) return; - List e = expand(PpSpan(textBuf)); + List e = expand(text); e = applyPragmaOperators(e, fileName); for(PpToken& t : e) { // report lines through any #line adjustment @@ -261,7 +264,6 @@ namespace rat::cc { t.line = adj >= 1 ? (U32)adj : 1; out.push_back(std::move(t)); } - textBuf.clear(); } B32 Preprocessor::condActive(const List& stack) { @@ -354,6 +356,10 @@ namespace rat::cc { return; List toks = lexFragment(source, intern(path)); + runTokens(path, toks); + } + + void Preprocessor::runTokens(const String& path, const List& toks) { if(!ok) return; @@ -363,7 +369,7 @@ namespace rat::cc { fileName = path; List stack; - List textBuf; + U64 textStart = 0; U64 i = 0, n = toks.size(); while(i < n && ok) { @@ -374,13 +380,12 @@ namespace rat::cc { i = j; const PpToken& first = toks[start]; - if(!isPunct(first, "#") || !first.bol) { - if(condActive(stack)) - textBuf.insert(textBuf.end(), toks.begin() + start, toks.begin() + j); + if(!isPunct(first, "#") || !first.bol) continue; - } - flush(textBuf); + if(condActive(stack)) + flush(PpSpan(toks.data() + textStart, toks.data() + start)); + textStart = j; if(start + 1 == j) // null directive continue; U32 phys = j < n ? toks[j].line : toks[start + 1].line + 1; @@ -390,7 +395,8 @@ namespace rat::cc { if(ok && !stack.empty()) fail(path + ": unterminated #if"); - flush(textBuf); + if(condActive(stack)) + flush(PpSpan(toks.data() + textStart, toks.data() + n)); lineDelta = savedDelta; fileName = savedFile; diff --git a/src/compiler/lex/preprocess_detail.h b/src/compiler/lex/preprocess_detail.h index 4c4b5b79..24e92151 100644 --- a/src/compiler/lex/preprocess_detail.h +++ b/src/compiler/lex/preprocess_detail.h @@ -20,18 +20,16 @@ namespace rat::cc { // string pool: equal spellings share one const String*, compare by pointer struct Interner { + struct Slot { + U64 hash = 0; + const String* str = nullptr; + }; std::deque store; - Map pool; - - const String* intern(std::string_view s) { - auto it = pool.find(s); - if(it != pool.end()) - return it->second; - store.emplace_back(s); - const String* p = &store.back(); - pool.emplace(std::string_view(*p), p); - return p; - } + List slots = List(4096); + U64 count = 0; + + const String* intern(std::string_view s); + void grow(); }; struct HideSet { @@ -39,13 +37,13 @@ namespace rat::cc { }; struct PpToken { - Pk kind = Pk::Eof; - const String* text = nullptr; // interned spelling - B32 spaceBefore = false; // had white space before it - B32 bol = false; // first token on its logical line - U32 line = 0; + const String* text = nullptr; // interned spelling const String* file = nullptr; // interned file name const HideSet* hide = nullptr; // interned hide set + U32 line = 0; + Pk kind = Pk::Eof; + U8 spaceBefore = false; // had white space before it + U8 bol = false; // first token on its logical line }; constexpr U32 kMaxIncludeDepth = 200; @@ -75,6 +73,10 @@ namespace rat::cc { U64 decodeTrigraph(const String& src, U64 p, C8& c); U64 newlineLen(const String& s, U64 i); + // 0x80 in each byte of w equal to b, 0 elsewhere + constexpr U64 kBytes1 = 0x0101010101010101ull; + U64 bytesEq(U64 w, C8 b); + // trigraph + splice + newline norm in one copy; sparse LineMarks void splice(const String& src, String& out, List& marks); @@ -183,6 +185,7 @@ namespace rat::cc { Map macros; List out; Set pragmaOnce; + Map> fileToks; struct SavedMacro { B32 defined; Macro macro; @@ -209,6 +212,7 @@ namespace rat::cc { const String* idAlias = interner.intern("alias"); const String* idAlias2 = interner.intern("__alias__"); const String* idAliasMark = interner.intern("__rat_alias__"); + const String* idPragma = interner.intern("_Pragma"); String err; B32 ok = true; I64 lineDelta = 0; @@ -278,7 +282,7 @@ namespace rat::cc { void doPragma(PpSpan rest, const String& path); String destringize(const String& lit); List applyPragmaOperators(List& toks, const String& path); - void flush(List& textBuf); + void flush(PpSpan text); static B32 condActive(const List& stack); void pushCond(const String& name, PpSpan rest, List& stack); B32 handleConditional(const String& name, PpSpan rest, List& stack); @@ -287,6 +291,7 @@ namespace rat::cc { // driver B32 run(const String& path, const String& source, String& errOut); void runFile(const String& path, const String& source); + void runTokens(const String& path, const List& toks); void installBuiltins(); void applyCommandLine(); String serialize(); diff --git a/src/compiler/lex/preprocess_lex.cpp b/src/compiler/lex/preprocess_lex.cpp index 168e04a2..ee9b1656 100644 --- a/src/compiler/lex/preprocess_lex.cpp +++ b/src/compiler/lex/preprocess_lex.cpp @@ -4,8 +4,45 @@ #include +#include "hash.h" + namespace rat::cc { namespace detail { + const String* Interner::intern(std::string_view s) { + U64 h = kFnvBasis; + for(C8 c : s) + hashMix(h, (U8)c); + U64 mask = slots.size() - 1; + for(U64 i = h & mask;; i = (i + 1) & mask) { + Slot& sl = slots[i]; + if(!sl.str) { + store.emplace_back(s); + sl.hash = h; + sl.str = &store.back(); + const String* p = sl.str; + if(++count * 2 > slots.size()) + grow(); + return p; + } + if(sl.hash == h && std::string_view(*sl.str) == s) + return sl.str; + } + } + + void Interner::grow() { + List old = std::move(slots); + slots = List(old.size() * 2); + U64 mask = slots.size() - 1; + for(const Slot& sl : old) { + if(!sl.str) + continue; + U64 i = sl.hash & mask; + while(slots[i].str) + i = (i + 1) & mask; + slots[i] = sl; + } + } + String unquote(const String& s) { return s.size() >= 2 ? s.substr(1, s.size() - 2) : s; } B32 isAbsPath(const String& s) { @@ -70,6 +107,12 @@ namespace rat::cc { return i + 1 < s.size() && s[i] == '\r' && s[i + 1] == '\n' ? 2 : 0; } + U64 bytesEq(U64 w, C8 b) { + constexpr U64 lo = 0x7F7F7F7F7F7F7F7Full; + U64 x = w ^ ((U64)(U8)b * kBytes1); + return ~(((x & lo) + lo) | x | lo); + } + // trigraph + splice + newline norm in one pass void splice(const String& src, String& out, List& marks) { U32 line = 1; @@ -79,6 +122,14 @@ namespace rat::cc { while(i < n) { // fast path: bulk-copy a run with no splice/trigraph/CR triggers U64 start = i; + while(i + 8 <= n) { + U64 w; + std::memcpy(&w, data + i, 8); + if(bytesEq(w, '\\') | bytesEq(w, '\r') | bytesEq(w, '?')) + break; + line += (U32)(((bytesEq(w, '\n') >> 7) * kBytes1) >> 56); + i += 8; + } while(i < n) { C8 c = data[i]; if(c == '\\' || c == '\r' || c == '?') @@ -223,10 +274,10 @@ namespace rat::cc { U64 PpLexer::wordEnd(U64 j) const { for(;;) { - if(U64 u = ucnLen(s, j)) - j += u; - else if(j < s.size() && isIdentCont(s[j])) + if(j < s.size() && isIdentCont(s[j])) ++j; + else if(U64 u = ucnLen(s, j)) + j += u; else return j; } diff --git a/src/compiler/lex/preprocess_macro.cpp b/src/compiler/lex/preprocess_macro.cpp index 9bde7f8c..327c711d 100644 --- a/src/compiler/lex/preprocess_macro.cpp +++ b/src/compiler/lex/preprocess_macro.cpp @@ -425,6 +425,7 @@ namespace rat::cc { for(U64 k = in.size(); k > 0; --k) work.push_back(in[k - 1]); List os; + os.reserve(in.size()); while(!work.empty()) { PpToken t = work.back(); work.pop_back();