diff --git a/src/backend/codegen/schedule.cpp b/src/backend/codegen/schedule.cpp index 7ed31172..4fd34a60 100644 --- a/src/backend/codegen/schedule.cpp +++ b/src/backend/codegen/schedule.cpp @@ -5,8 +5,30 @@ #include "ir/node.h" namespace rat { - Schedule::Schedule(const Function& fn, Mode mode) + Schedule::Schedule(const Function& fn) : fn(fn) { + buildBlocks(); + List work; + for(Node* n : fn) + if(isFloating(n)) + work.push_back(n); + List early(fn.idBound(), -1); + scheduleEarly(work, early); + markGuarded(work); + scheduleLate(work, early); + buildBlockLists(); + } + + Schedule::Schedule(const Function& fn, const List& loads) + : fn(fn) { + buildBlocks(); + List work = floatingCone(loads); + List early(fn.idBound(), -1); + scheduleEarly(work, early); + placeLoads(work, early); + } + + void Schedule::buildBlocks() { U32 count = fn.idBound(); headIndex.assign(count, -1); nodeBlock.assign(count, -1); @@ -17,19 +39,25 @@ namespace rat { computeDominators(); computeLoops(); computeHoistBounds(); + } + + List Schedule::floatingCone(const List& loads) const { + List seen(fn.idBound(), 0); List work; - for(Node* n : fn) - if(isFloating(n)) - work.push_back(n); - List early(count, -1); - scheduleEarly(work, early); - if(mode == Mode::Loads) { - placeLoads(work, early); - return; - } - markGuarded(work); - scheduleLate(work, early); - buildBlockLists(); + for(LoadNode* l : loads) { + seen[l->getId()] = 1; + work.push_back(l); + } + for(U32 i = 0; i < work.size(); ++i) + for(U32 k = 0, e = work[i]->getInputCount(); k < e; ++k) { + Node* in = work[i]->getInput(k); + if(in && isFloating(in) && !seen[in->getId()]) { + seen[in->getId()] = 1; + work.push_back(in); + } + } + std::reverse(work.begin(), work.end()); // inputs first, the early fixpoint settles sooner + return work; } I32 Schedule::numBlocks() const { return (I32)blocks.size(); } diff --git a/src/backend/codegen/schedule.h b/src/backend/codegen/schedule.h index 4022ef57..c7b80c92 100644 --- a/src/backend/codegen/schedule.h +++ b/src/backend/codegen/schedule.h @@ -51,12 +51,9 @@ namespace rat { List nodes; // scheduled compute nodes, in emit order }; - enum class Mode { - Full, - Loads, // blockOf(load), dominators and loop depth - }; - - explicit Schedule(const Function& fn, Mode mode = Mode::Full); + explicit Schedule(const Function& fn); + // blockOf for the given loads only, dominators and loop depth + Schedule(const Function& fn, const List& loads); I32 numBlocks() const; const Block& block(I32 b) const; @@ -82,6 +79,7 @@ namespace rat { U32 succCount(I32 b) const; I32 succAt(I32 b, U32 i) const; + void buildBlocks(); void collectHeads(); void buildCFG(); BlockEnd walkBlock(I32 b); @@ -95,6 +93,7 @@ namespace rat { void scheduleLate(const List& work, const List& early); I32 lateBlock(Node* n) const; void placeLoads(const List& work, const List& early); + List floatingCone(const List& loads) const; B32 place(Node* n, I32 late, const List& early); I32 listedBlock(const Node* n) const; void buildBlockLists(); diff --git a/src/backend/ir/function.cpp b/src/backend/ir/function.cpp index ebd61b9d..f93df0d7 100644 --- a/src/backend/ir/function.cpp +++ b/src/backend/ir/function.cpp @@ -372,6 +372,22 @@ namespace rat { void Function::retVoid() { ret(nullptr); } + B32 Function::isCleanFor(const void* pass) const { + for(const auto& [p, v] : cleanAt) + if(p == pass) + return v == version; + return false; + } + + void Function::markCleanFor(const void* pass) { + for(auto& [p, v] : cleanAt) + if(p == pass) { + v = version; + return; + } + cleanAt.push_back({pass, version}); + } + B32 Function::hasReturn() const { for(Node* n : *this) if(isa(n)) @@ -396,6 +412,8 @@ namespace rat { } U32 Function::eliminateDeadNodes(B32 includeControl) { + if(!includeControl && deadFreeAt == version) + return 0; // unchanged since a sweep, which leaves no dead node List deadMark(nextId, 0); // id-indexed List work(nodes.begin(), nodes.end()); U32 count = 0; @@ -415,6 +433,7 @@ namespace rat { touch(); eraseMarked(deadMark); } + deadFreeAt = version; return count; } diff --git a/src/backend/ir/function.h b/src/backend/ir/function.h index 46fa39a0..6c0a8b85 100644 --- a/src/backend/ir/function.h +++ b/src/backend/ir/function.h @@ -172,11 +172,8 @@ namespace rat { U64 getVersion() const { return version; } void touch() { ++version; } - B32 isCleanFor(const void* pass) const { - auto it = cleanAt.find(pass); - return it != cleanAt.end() && it->second == version; - } - void markCleanFor(const void* pass) { cleanAt[pass] = version; } + B32 isCleanFor(const void* pass) const; + void markCleanFor(const void* pass); B32 hasReturn() const; U32 eliminateDeadNodes(B32 includeControl = false); @@ -228,8 +225,9 @@ namespace rat { List paramCache; // pass state - U64 version = 0; // bumped on every mutation - Map cleanAt; + U64 version = 0; // bumped on every mutation + U64 deadFreeAt = ~(U64)0; // version at the last dead-node sweep + List> cleanAt; // builder state Block* cur = nullptr; // current insertion block diff --git a/src/backend/pass/opt/fold.cpp b/src/backend/pass/opt/fold.cpp index 98363191..9d9d1343 100644 --- a/src/backend/pass/opt/fold.cpp +++ b/src/backend/pass/opt/fold.cpp @@ -704,7 +704,7 @@ namespace rat { fresh = 0; work.clear(); work.reserve(fn.size()); - queued.assign(fn.size(), 0); + queued.assign(fn.idBound(), 0); for(Node* n : fn) { if(n->getId() >= fresh) fresh = n->getId() + 1; diff --git a/src/backend/pass/opt/gvn.cpp b/src/backend/pass/opt/gvn.cpp index 06b06cb7..07df7608 100644 --- a/src/backend/pass/opt/gvn.cpp +++ b/src/backend/pass/opt/gvn.cpp @@ -40,10 +40,16 @@ namespace rat { B32 GVNPass::isPureValue(Node* n) { Opcode op = n->getOpcode(); - if(Schedule::mayTrap(n)) - return false; - return op == Opcode::Constant || op == Opcode::Global || isArithmeticOpcode(op) || - isVectorUtilOpcode(op); + B32 value = op == Opcode::Constant || op == Opcode::Global || isArithmeticOpcode(op) || + isVectorUtilOpcode(op); + return value && !Schedule::mayTrap(n); + } + + B32 GVNPass::hasKeyedUser(Node* n) const { + for(Node* u : n->getUsers()) + if(keyed[u->getId()]) + return true; + return false; } const C8* GVNPass::name() const { return "gvn"; } @@ -61,9 +67,11 @@ namespace rat { B32 changed = true; while(changed) { + U32 before = removed; changed = false; for(U32 i = 0; i < cap; ++i) slots[i].val = nullptr; + keyed.assign(fn.idBound(), 0); U32 filled = 0; for(Node* n : fn) { if(!isPureValue(n) || !n->hasUsers()) @@ -71,6 +79,7 @@ namespace rat { detail::GVNKey key; if(!detail::makeKey(n, key)) continue; + keyed[n->getId()] = 1; U32 i = (U32)hasher(key) & mask; while(slots[i].val && !(slots[i].key == key)) i = (i + 1) & mask; @@ -78,13 +87,15 @@ namespace rat { slots[i].key = key; slots[i].val = n; ++filled; - if(filled * 2 >= cap) + if(filled * 2 >= cap) { + changed = removed != before; break; + } } else { // n is a duplicate of the earlier representative; redirect its uses + changed = changed || hasKeyedUser(n); n->replaceAllUsesWith(slots[i].val); ++removed; - changed = true; } } } diff --git a/src/backend/pass/opt/gvn.h b/src/backend/pass/opt/gvn.h index 43bb090e..6ad69a6a 100644 --- a/src/backend/pass/opt/gvn.h +++ b/src/backend/pass/opt/gvn.h @@ -53,12 +53,14 @@ namespace rat { U32 runOnFunction(Function& fn, const TargetInfo& target) override; private: static B32 isPureValue(Node* n); + B32 hasKeyedUser(Node* n) const; private: struct Slot { detail::GVNKey key; Node* val = nullptr; }; List slots; + List keyed; // id -> hashed in the current sweep }; } // namespace rat diff --git a/src/backend/pass/opt/inline.cpp b/src/backend/pass/opt/inline.cpp index e62ad15f..a7cd8520 100644 --- a/src/backend/pass/opt/inline.cpp +++ b/src/backend/pass/opt/inline.cpp @@ -184,10 +184,7 @@ namespace rat { } } - void InlinePass::forgetCycles() { - for(auto& [fn, info] : infos) - info.cyclic = -1; - } + void InlinePass::forgetCycles() { ++cycleGen; } // refresh the rows of functions mutated since their row was built void InlinePass::syncCallGraph(Module& m) { @@ -227,9 +224,10 @@ namespace rat { B32 InlinePass::isCyclic(Function* fn) { Info& info = infos[fn]; - if(info.cyclic < 0) { + if(info.cyclicAt != cycleGen) { ++visitCur; info.cyclic = reaches(&info, &info); + info.cyclicAt = cycleGen; } return info.cyclic; } @@ -245,6 +243,8 @@ namespace rat { B32 InlinePass::shouldInline(const Function& caller, CallNode* call, Function* callee) { if(!callee || callee == &caller) return false; // missing or directly recursive + if(callee->size() > kInlineNodeBudget) + return false; if(callee->getAttrs().noInline) return false; // opted out via __attribute__((noinline)) if(isCyclic(callee)) @@ -261,7 +261,7 @@ namespace rat { for(U32 i = 0, e = call->getArgCount(); i < e; ++i) if(call->getArg(i)->getType() != callee->getParamType(i)) return false; - return callee->size() <= kInlineNodeBudget; + return true; } B32 InlinePass::run(Module& module, const TargetInfo& target) { diff --git a/src/backend/pass/opt/inline.h b/src/backend/pass/opt/inline.h index 20bd8d52..1d0fd626 100644 --- a/src/backend/pass/opt/inline.h +++ b/src/backend/pass/opt/inline.h @@ -37,7 +37,8 @@ namespace rat { U64 quietAt = 0; // stamp of the last run that inlined nothing, 0 = none U32 firstSize = 0; // caller size when first seen, bounds growth per module run U32 visit = 0; // dfs stamp - I8 cyclic = -1; // cached isCyclic, -1 = unknown + B32 cyclic = false; // cached isCyclic + U32 cyclicAt = 0; }; static constexpr U64 kNoRow = ~(U64)0; @@ -73,6 +74,7 @@ namespace rat { Map infos; Map byName; U32 visitCur = 0; + U32 cycleGen = 1; // bumped when a call edge changes // scratch reused across call sites List cloneMap; // callee node id -> caller node diff --git a/src/backend/pass/opt/memory_opt.cpp b/src/backend/pass/opt/memory_opt.cpp index 12ed5543..a7aa3637 100644 --- a/src/backend/pass/opt/memory_opt.cpp +++ b/src/backend/pass/opt/memory_opt.cpp @@ -61,8 +61,6 @@ namespace rat { } U32 MemoryOptPass::cseLoads(Function& fn, const AliasAnalysis& aa) { - Schedule sched(fn, Schedule::Mode::Loads); - buckets.clear(); for(LoadNode* l : loads) { if(!l->hasUsers()) @@ -72,7 +70,14 @@ namespace rat { continue; // opaque address / unknown size: not provably CSE buckets[BucketKey{defs[l->getId()], l->getType(), key}].push_back(l); } - + paired.clear(); + for(auto& kv : buckets) + if(kv.second.size() >= 2) + paired.insert(paired.end(), kv.second.begin(), kv.second.end()); + if(paired.empty()) + return 0; // nothing to CSE, skip the schedule + + Schedule sched(fn, paired); U32 removed = 0; for(auto& kv : buckets) { List& group = kv.second; diff --git a/src/backend/pass/opt/memory_opt.h b/src/backend/pass/opt/memory_opt.h index 12a51216..5c04a187 100644 --- a/src/backend/pass/opt/memory_opt.h +++ b/src/backend/pass/opt/memory_opt.h @@ -70,6 +70,7 @@ namespace rat { }; private: List loads; + List paired; List defs; List chainHops; std::unordered_map, BucketKeyHash> buckets; diff --git a/src/backend/pass/opt/sccp.cpp b/src/backend/pass/opt/sccp.cpp index 4c2c485c..db9dd6ec 100644 --- a/src/backend/pass/opt/sccp.cpp +++ b/src/backend/pass/opt/sccp.cpp @@ -58,8 +58,9 @@ namespace rat { } void SCCPPass::pushPhis(RegionNode* r) { - for(PhiNode* phi : usersOfType(r)) - ssaWork.push_back(phi); + for(Node* u : r->getUsers()) + if(isa(u)) + ssaWork.push_back(u); } void SCCPPass::evalIf(IfNode* iff) { diff --git a/src/backend/pass/opt/simplify_cfg.cpp b/src/backend/pass/opt/simplify_cfg.cpp index 1664dacf..450acd63 100644 --- a/src/backend/pass/opt/simplify_cfg.cpp +++ b/src/backend/pass/opt/simplify_cfg.cpp @@ -7,16 +7,17 @@ namespace rat { void SimplifyCFGPass::reachableControl(Function& fn) { - reach.clear(); - reach.insert(fn.getStart()); + reach.assign(fn.idBound(), 0); + reach[fn.getStart()->getId()] = 1; stack.clear(); if(Node* e = fn.getStart()->projection(StartNode::controlProjIndex())) stack.push_back(e); while(!stack.empty()) { Node* n = stack.back(); stack.pop_back(); - if(!reach.insert(n).second) + if(reach[n->getId()]) continue; + reach[n->getId()] = 1; for(Node* u : n->getUsers()) { if(isControlNode(u)) { stack.push_back(u); @@ -29,6 +30,10 @@ namespace rat { } } + B32 SimplifyCFGPass::reached(Node* n) const { + return n && n->getId() < reach.size() && reach[n->getId()]; + } + void SimplifyCFGPass::collectPhis(Node* region) { phis.clear(); for(Node* u : region->getUsers()) @@ -230,7 +235,7 @@ namespace rat { if(StopNode* stop = fn.getStop()) for(U32 i = stop->getInputCount(); i-- > 0;) { Node* r = stop->getInput(i); - if(r && !reach.count(r)) { + if(r && !reached(r)) { stop->removeInput(i); ++changed; } @@ -240,9 +245,9 @@ namespace rat { continue; B32 dead = false; if(isControlNode(n)) - dead = !reach.count(n); + dead = !reached(n); else if(Node* ci = n->getControlInput()) - dead = ci != fn.getStart() && !reach.count(ci); + dead = ci != fn.getStart() && !reached(ci); if(!dead) continue; if(n->getInputCount() > 0) { @@ -257,10 +262,10 @@ namespace rat { U32 changed = 0; for(Node* n : regions) { RegionNode* r = cast(n); - if(!reach.count(r)) + if(!reached(r)) continue; for(U32 i = r->getPredecessorCount(); i-- > 0;) - if(!reach.count(r->getPredecessor(i))) { + if(!reached(r->getPredecessor(i))) { removePred(r, i); ++changed; } @@ -289,7 +294,7 @@ namespace rat { U32 changed = 0; for(Node* n : regions) { RegionNode* r = cast(n); - if(!reach.count(r) || r->getPredecessorCount() != 1) + if(!reached(r) || r->getPredecessorCount() != 1) continue; collectPhis(r); for(PhiNode* phi : phis) diff --git a/src/backend/pass/opt/simplify_cfg.h b/src/backend/pass/opt/simplify_cfg.h index 0b40656d..b0fe7183 100644 --- a/src/backend/pass/opt/simplify_cfg.h +++ b/src/backend/pass/opt/simplify_cfg.h @@ -38,6 +38,7 @@ namespace rat { U32 collapseRegions(); void reachableControl(Function& fn); + B32 reached(Node* n) const; void collectPhis(Node* region); static void removePred(RegionNode* r, U32 i); static void foldIf(Function& fn, IfNode* iff, B32 thenTaken); @@ -52,7 +53,7 @@ namespace rat { private: List ifs; List regions; - Set reach; // reachable control + List reach; // id -> reachable control List phis; // walk scratch List stack; diff --git a/src/backend/pass/opt/slp/slp_driver.cpp b/src/backend/pass/opt/slp/slp_driver.cpp index 327064d5..4c7c44c4 100644 --- a/src/backend/pass/opt/slp/slp_driver.cpp +++ b/src/backend/pass/opt/slp/slp_driver.cpp @@ -161,6 +161,9 @@ namespace rat { LoadNode* l = dyn_cast(n); if(!l) continue; + StoreNode* above = dyn_cast(l->getMemory()); + if(!above || above->getControl() != l->getControl()) + continue; U32 lsz = aa.getAccessSize(l); if(!lsz) continue; @@ -324,6 +327,8 @@ namespace rat { List slp::Slp::buildSegments(const Map& cand) { List segments; for(Node* n : fn) { + if(!isa(n)) + continue; // candidates are stores auto headIt = cand.find(n); if(headIt == cand.end()) continue; diff --git a/src/backend/pass/pass.cpp b/src/backend/pass/pass.cpp index 42d798dc..7f651baf 100644 --- a/src/backend/pass/pass.cpp +++ b/src/backend/pass/pass.cpp @@ -10,12 +10,12 @@ namespace rat { U32 changed = 0; for(Function* fn : module) { B32 skippable = onlyReadsFunction(); - if(skippable && fn->isCleanFor(this)) + if(skippable && fn->isCleanFor(name())) continue; if(runOnFunction(*fn, target)) ++changed; else if(skippable) - fn->markCleanFor(this); + fn->markCleanFor(name()); } return changed != 0; }