Nella/opt overhead

src/backend/codegen/schedule.cpp

@@ -5,8 +5,30 @@
 #include "ir/node.h"
 
 namespace rat {
-	Schedule::Schedule(const Function& fn, Mode mode)
+	Schedule::Schedule(const Function& fn)
 	: fn(fn) {
+		buildBlocks();
+		List<Node*> work;
+		for(Node* n : fn)
+			if(isFloating(n))
+				work.push_back(n);
+		List<I32> early(fn.idBound(), -1);
+		scheduleEarly(work, early);
+		markGuarded(work);
+		scheduleLate(work, early);
+		buildBlockLists();
+	}
+
+	Schedule::Schedule(const Function& fn, const List<LoadNode*>& loads)
+	: fn(fn) {
+		buildBlocks();
+		List<Node*> work = floatingCone(loads);
+		List<I32> early(fn.idBound(), -1);
+		scheduleEarly(work, early);
+		placeLoads(work, early);
+	}
+
+	void Schedule::buildBlocks() {
 		U32 count = fn.idBound();
 		headIndex.assign(count, -1);
 		nodeBlock.assign(count, -1);
@@ -17,19 +39,25 @@ namespace rat {
 		computeDominators();
 		computeLoops();
 		computeHoistBounds();
+	}
+
+	List<Node*> Schedule::floatingCone(const List<LoadNode*>& loads) const {
+		List<U8> seen(fn.idBound(), 0);
 		List<Node*> work;
-		for(Node* n : fn)
-			if(isFloating(n))
-				work.push_back(n);
-		List<I32> early(count, -1);
-		scheduleEarly(work, early);
-		if(mode == Mode::Loads) {
-			placeLoads(work, early);
-			return;
-		}
-		markGuarded(work);
-		scheduleLate(work, early);
-		buildBlockLists();
+		for(LoadNode* l : loads) {
+			seen[l->getId()] = 1;
+			work.push_back(l);
+		}
+		for(U32 i = 0; i < work.size(); ++i)
+			for(U32 k = 0, e = work[i]->getInputCount(); k < e; ++k) {
+				Node* in = work[i]->getInput(k);
+				if(in && isFloating(in) && !seen[in->getId()]) {
+					seen[in->getId()] = 1;
+					work.push_back(in);
+				}
+			}
+		std::reverse(work.begin(), work.end()); // inputs first, the early fixpoint settles sooner
+		return work;
 	}
 
 	I32 Schedule::numBlocks() const { return (I32)blocks.size(); }

src/backend/codegen/schedule.h

@@ -51,12 +51,9 @@ namespace rat {
 			List<Node*> nodes;	 // scheduled compute nodes, in emit order
 		};
 
-		enum class Mode {
-			Full,
-			Loads, // blockOf(load), dominators and loop depth
-		};
-
-		explicit Schedule(const Function& fn, Mode mode = Mode::Full);
+		explicit Schedule(const Function& fn);
+		// blockOf for the given loads only, dominators and loop depth
+		Schedule(const Function& fn, const List<LoadNode*>& loads);
 
 		I32 numBlocks() const;
 		const Block& block(I32 b) const;
@@ -82,6 +79,7 @@ namespace rat {
 		U32 succCount(I32 b) const;
 		I32 succAt(I32 b, U32 i) const;
 
+		void buildBlocks();
 		void collectHeads();
 		void buildCFG();
 		BlockEnd walkBlock(I32 b);
@@ -95,6 +93,7 @@ namespace rat {
 		void scheduleLate(const List<Node*>& work, const List<I32>& early);
 		I32 lateBlock(Node* n) const;
 		void placeLoads(const List<Node*>& work, const List<I32>& early);
+		List<Node*> floatingCone(const List<LoadNode*>& loads) const;
 		B32 place(Node* n, I32 late, const List<I32>& early);
 		I32 listedBlock(const Node* n) const;
 		void buildBlockLists();

src/backend/ir/function.cpp

@@ -372,6 +372,22 @@ namespace rat {
 
 	void Function::retVoid() { ret(nullptr); }
 
+	B32 Function::isCleanFor(const void* pass) const {
+		for(const auto& [p, v] : cleanAt)
+			if(p == pass)
+				return v == version;
+		return false;
+	}
+
+	void Function::markCleanFor(const void* pass) {
+		for(auto& [p, v] : cleanAt)
+			if(p == pass) {
+				v = version;
+				return;
+			}
+		cleanAt.push_back({pass, version});
+	}
+
 	B32 Function::hasReturn() const {
 		for(Node* n : *this)
 			if(isa<ReturnNode>(n))
@@ -396,6 +412,8 @@ namespace rat {
 	}
 
 	U32 Function::eliminateDeadNodes(B32 includeControl) {
+		if(!includeControl && deadFreeAt == version)
+			return 0;										// unchanged since a sweep, which leaves no dead node
 		List<U8> deadMark(nextId, 0); // id-indexed
 		List<Node*> work(nodes.begin(), nodes.end());
 		U32 count = 0;
@@ -415,6 +433,7 @@ namespace rat {
 			touch();
 			eraseMarked(deadMark);
 		}
+		deadFreeAt = version;
 		return count;
 	}
 

src/backend/ir/function.h

@@ -172,11 +172,8 @@ namespace rat {
 
 		U64 getVersion() const { return version; }
 		void touch() { ++version; }
-		B32 isCleanFor(const void* pass) const {
-			auto it = cleanAt.find(pass);
-			return it != cleanAt.end() && it->second == version;
-		}
-		void markCleanFor(const void* pass) { cleanAt[pass] = version; }
+		B32 isCleanFor(const void* pass) const;
+		void markCleanFor(const void* pass);
 		B32 hasReturn() const;
 
 		U32 eliminateDeadNodes(B32 includeControl = false);
@@ -228,8 +225,9 @@ namespace rat {
 		List<Node*> paramCache;
 
 		// pass state
-		U64 version = 0; // bumped on every mutation
-		Map<const void*, U64> cleanAt;
+		U64 version = 0;					// bumped on every mutation
+		U64 deadFreeAt = ~(U64)0; // version at the last dead-node sweep
+		List<Pair<const void*, U64>> cleanAt;
 
 		// builder state
 		Block* cur = nullptr; // current insertion block

src/backend/pass/opt/fold.cpp

@@ -704,7 +704,7 @@ namespace rat {
 		fresh = 0;
 		work.clear();
 		work.reserve(fn.size());
-		queued.assign(fn.size(), 0);
+		queued.assign(fn.idBound(), 0);
 		for(Node* n : fn) {
 			if(n->getId() >= fresh)
 				fresh = n->getId() + 1;

src/backend/pass/opt/gvn.cpp

@@ -40,10 +40,16 @@ namespace rat {
 
 	B32 GVNPass::isPureValue(Node* n) {
 		Opcode op = n->getOpcode();
-		if(Schedule::mayTrap(n))
-			return false;
-		return op == Opcode::Constant || op == Opcode::Global || isArithmeticOpcode(op) ||
-					 isVectorUtilOpcode(op);
+		B32 value = op == Opcode::Constant || op == Opcode::Global || isArithmeticOpcode(op) ||
+								isVectorUtilOpcode(op);
+		return value && !Schedule::mayTrap(n);
+	}
+
+	B32 GVNPass::hasKeyedUser(Node* n) const {
+		for(Node* u : n->getUsers())
+			if(keyed[u->getId()])
+				return true;
+		return false;
 	}
 
 	const C8* GVNPass::name() const { return "gvn"; }
@@ -61,30 +67,35 @@ namespace rat {
 
 		B32 changed = true;
 		while(changed) {
+			U32 before = removed;
 			changed = false;
 			for(U32 i = 0; i < cap; ++i)
 				slots[i].val = nullptr;
+			keyed.assign(fn.idBound(), 0);
 			U32 filled = 0;
 			for(Node* n : fn) {
 				if(!isPureValue(n) || !n->hasUsers())
 					continue;
 				detail::GVNKey key;
 				if(!detail::makeKey(n, key))
 					continue;
+				keyed[n->getId()] = 1;
 				U32 i = (U32)hasher(key) & mask;
 				while(slots[i].val && !(slots[i].key == key))
 					i = (i + 1) & mask;
 				if(!slots[i].val) {
 					slots[i].key = key;
 					slots[i].val = n;
 					++filled;
-					if(filled * 2 >= cap)
+					if(filled * 2 >= cap) {
+						changed = removed != before;
 						break;
+					}
 				} else {
 					// n is a duplicate of the earlier representative; redirect its uses
+					changed = changed || hasKeyedUser(n);
 					n->replaceAllUsesWith(slots[i].val);
 					++removed;
-					changed = true;
 				}
 			}
 		}

src/backend/pass/opt/gvn.h

@@ -53,12 +53,14 @@ namespace rat {
 		U32 runOnFunction(Function& fn, const TargetInfo& target) override;
 	private:
 		static B32 isPureValue(Node* n);
+		B32 hasKeyedUser(Node* n) const;
 	private:
 		struct Slot {
 			detail::GVNKey key;
 			Node* val = nullptr;
 		};
 		List<Slot> slots;
+		List<U8> keyed; // id -> hashed in the current sweep
 	};
 } // namespace rat
 

src/backend/pass/opt/inline.cpp

@@ -184,10 +184,7 @@ namespace rat {
 		}
 	}
 
-	void InlinePass::forgetCycles() {
-		for(auto& [fn, info] : infos)
-			info.cyclic = -1;
-	}
+	void InlinePass::forgetCycles() { ++cycleGen; }
 
 	// refresh the rows of functions mutated since their row was built
 	void InlinePass::syncCallGraph(Module& m) {
@@ -227,9 +224,10 @@ namespace rat {
 
 	B32 InlinePass::isCyclic(Function* fn) {
 		Info& info = infos[fn];
-		if(info.cyclic < 0) {
+		if(info.cyclicAt != cycleGen) {
 			++visitCur;
 			info.cyclic = reaches(&info, &info);
+			info.cyclicAt = cycleGen;
 		}
 		return info.cyclic;
 	}
@@ -245,6 +243,8 @@ namespace rat {
 	B32 InlinePass::shouldInline(const Function& caller, CallNode* call, Function* callee) {
 		if(!callee || callee == &caller)
 			return false; // missing or directly recursive
+		if(callee->size() > kInlineNodeBudget)
+			return false;
 		if(callee->getAttrs().noInline)
 			return false; // opted out via __attribute__((noinline))
 		if(isCyclic(callee))
@@ -261,7 +261,7 @@ namespace rat {
 		for(U32 i = 0, e = call->getArgCount(); i < e; ++i)
 			if(call->getArg(i)->getType() != callee->getParamType(i))
 				return false;
-		return callee->size() <= kInlineNodeBudget;
+		return true;
 	}
 
 	B32 InlinePass::run(Module& module, const TargetInfo& target) {

src/backend/pass/opt/inline.h

@@ -37,7 +37,8 @@ namespace rat {
 			U64 quietAt = 0;			// stamp of the last run that inlined nothing, 0 = none
 			U32 firstSize = 0;		// caller size when first seen, bounds growth per module run
 			U32 visit = 0;				// dfs stamp
-			I8 cyclic = -1;				// cached isCyclic, -1 = unknown
+			B32 cyclic = false;		// cached isCyclic
+			U32 cyclicAt = 0;
 		};
 		static constexpr U64 kNoRow = ~(U64)0;
 
@@ -73,6 +74,7 @@ namespace rat {
 		Map<const Function*, Info> infos;
 		Map<String, Function*> byName;
 		U32 visitCur = 0;
+		U32 cycleGen = 1; // bumped when a call edge changes
 
 		// scratch reused across call sites
 		List<Node*> cloneMap; // callee node id -> caller node

src/backend/pass/opt/memory_opt.cpp

@@ -61,8 +61,6 @@ namespace rat {
 	}
 
 	U32 MemoryOptPass::cseLoads(Function& fn, const AliasAnalysis& aa) {
-		Schedule sched(fn, Schedule::Mode::Loads);
-
 		buckets.clear();
 		for(LoadNode* l : loads) {
 			if(!l->hasUsers())
@@ -72,7 +70,14 @@ namespace rat {
 				continue; // opaque address / unknown size: not provably CSE
 			buckets[BucketKey{defs[l->getId()], l->getType(), key}].push_back(l);
 		}
-
+		paired.clear();
+		for(auto& kv : buckets)
+			if(kv.second.size() >= 2)
+				paired.insert(paired.end(), kv.second.begin(), kv.second.end());
+		if(paired.empty())
+			return 0; // nothing to CSE, skip the schedule
+
+		Schedule sched(fn, paired);
 		U32 removed = 0;
 		for(auto& kv : buckets) {
 			List<LoadNode*>& group = kv.second;

src/backend/pass/opt/memory_opt.h

@@ -70,6 +70,7 @@ namespace rat {
 		};
 	private:
 		List<LoadNode*> loads;
+		List<LoadNode*> paired;
 		List<Node*> defs;
 		List<U32> chainHops;
 		std::unordered_map<BucketKey, List<LoadNode*>, BucketKeyHash> buckets;

src/backend/pass/opt/sccp.cpp

@@ -58,8 +58,9 @@ namespace rat {
 	}
 
 	void SCCPPass::pushPhis(RegionNode* r) {
-		for(PhiNode* phi : usersOfType<PhiNode>(r))
-			ssaWork.push_back(phi);
+		for(Node* u : r->getUsers())
+			if(isa<PhiNode>(u))
+				ssaWork.push_back(u);
 	}
 
 	void SCCPPass::evalIf(IfNode* iff) {