From ab71ae3205b0a40ac05279c1583d5abe064d6caa Mon Sep 17 00:00:00 2001 From: nella Date: Mon, 12 Oct 2026 16:47:23 +0200 Subject: [PATCH 01/16] Detach edges in constant time. --- src/backend/ir/function.h | 1 + src/backend/ir/node.cpp | 63 ++++++++++++++++++++++++++----------- src/backend/ir/node.h | 13 +++++--- src/backend/pass/verify.cpp | 5 ++- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/backend/ir/function.h b/src/backend/ir/function.h index 0b5d12de..153e1cf7 100644 --- a/src/backend/ir/function.h +++ b/src/backend/ir/function.h @@ -185,6 +185,7 @@ namespace rat { private: U32 allocateId(); Node** allocEdges(U32 count) { return arena.makeArray(count); } + U32* allocSlots(U32 count) { return arena.makeArray(count); } const C8* internString(const C8* s, U64 len) { return arena.internString(s, len); } B32 isDeadNode(Node* n, B32 includeControl) const; U32 eraseMarked(const List& mark); diff --git a/src/backend/ir/node.cpp b/src/backend/ir/node.cpp index 61723ec7..8c809926 100644 --- a/src/backend/ir/node.cpp +++ b/src/backend/ir/node.cpp @@ -12,24 +12,31 @@ namespace rat { if(inputCount) { inputCap = inputCount; this->inputs = fn.allocEdges(inputCap); + inputPos = fn.allocSlots(inputCap); std::memcpy(this->inputs, inputs.data(), sizeof(Node*) * inputCount); for(U32 i = 0; i < inputCount; ++i) if(Node* in = this->inputs[i]) - in->addUser(this); + in->addUser(this, i); } } const C8* Node::getMnemonic() const { return getOpcodeMnemonic(op); } - void Node::addUser(Node* user) { + void Node::addUser(Node* user, U32 slot) { if(userCount == userCap) { U32 cap = userCap ? userCap * 2 : 4; Node** grown = fn->allocEdges(cap); - if(userCount) + U32* grownSlots = fn->allocSlots(cap); + if(userCount) { std::memcpy(grown, users, sizeof(Node*) * userCount); + std::memcpy(grownSlots, userSlots, sizeof(U32) * userCount); + } users = grown; + userSlots = grownSlots; userCap = cap; } + user->inputPos[slot] = userCount; + userSlots[userCount] = slot; users[userCount++] = user; } @@ -38,9 +45,13 @@ namespace rat { return; U32 cap = std::max(inputCap ? inputCap * 2 : 4, needed); Node** grown = fn->allocEdges(cap); - if(inputCount) + U32* grownPos = fn->allocSlots(cap); + if(inputCount) { std::memcpy(grown, inputs, sizeof(Node*) * inputCount); + std::memcpy(grownPos, inputPos, sizeof(U32) * inputCount); + } inputs = grown; + inputPos = grownPos; inputCap = cap; } @@ -49,36 +60,46 @@ namespace rat { growInputs(inputCount + 1); inputs[inputCount++] = value; if(value) - value->addUser(this); + value->addUser(this, inputCount - 1); } - void Node::removeUser(Node* user) { - // remove a single occurrence: each using operand contributes one entry; - for(U32 i = userCount; i-- > 0;) - if(users[i] == user) { - users[i] = users[--userCount]; - return; - } + void Node::detachInput(U32 index) { + Node* def = inputs[index]; + U32 at = inputPos[index]; + U32 last = --def->userCount; + if(at == last) + return; + Node* moved = def->users[last]; + U32 slot = def->userSlots[last]; + def->users[at] = moved; + def->userSlots[at] = slot; + moved->inputPos[slot] = at; } void Node::setInput(U32 index, Node* value) { + assert(index < inputCount && "setInput on a slot that is not live"); Node* old = inputs[index]; if(old == value) return; fn->touch(); if(old) - old->removeUser(this); + detachInput(index); inputs[index] = value; if(value) - value->addUser(this); + value->addUser(this, index); } void Node::removeInput(U32 index) { + assert(index < inputCount && "removeInput on a slot that is not live"); fn->touch(); if(inputs[index]) - inputs[index]->removeUser(this); - for(U32 i = index + 1; i < inputCount; ++i) + detachInput(index); + for(U32 i = index + 1; i < inputCount; ++i) { inputs[i - 1] = inputs[i]; + inputPos[i - 1] = inputPos[i]; + if(inputs[i - 1]) + inputs[i - 1]->userSlots[inputPos[i - 1]] = i - 1; + } --inputCount; } @@ -87,7 +108,7 @@ namespace rat { fn->touch(); for(U32 i = 0; i < inputCount; ++i) if(inputs[i]) - inputs[i]->removeUser(this); + detachInput(i); inputCount = 0; } @@ -102,12 +123,18 @@ namespace rat { if(user->inputs[i] == this) { user->inputs[i] = value; if(value) - value->addUser(user); + value->addUser(user, i); } } userCount = 0; } + B32 Node::inputEdgeIndexed(U32 index) const { + const Node* in = inputs[index]; + U32 at = inputPos[index]; + return at < in->userCount && in->users[at] == this && in->userSlots[at] == index; + } + B32 Node::isCFG() const { return getOpcodeInfo(op).isCFG; } B32 Node::hasSideEffects() const { return getOpcodeInfo(op).hasSideEffects; } diff --git a/src/backend/ir/node.h b/src/backend/ir/node.h index 4c9c3f4a..326a6243 100644 --- a/src/backend/ir/node.h +++ b/src/backend/ir/node.h @@ -64,22 +64,27 @@ namespace rat { B32 hasSideEffects() const; B32 isCommutative() const; + // the users entry of input index points back at this node and slot + B32 inputEdgeIndexed(U32 index) const; + // the control anchor for anchored nodes Node* getControlInput() const; // find this node's projection or null ProjNode* projection(U32 index) const; protected: - void removeUser(Node* user); - void addUser(Node* user); + void detachInput(U32 index); // drop the users entry of a live slot + void addUser(Node* user, U32 slot); void growInputs(U32 needed); Opcode op; Type* ty; U32 id; // unique per function Function* fn; - Node** inputs = nullptr; // ordered defs; may contain null placeholders - Node** users = nullptr; // reverse edges; one entry per using operand + Node** inputs = nullptr; // ordered defs; may contain null placeholders + Node** users = nullptr; // reverse edges; one entry per using operand + U32* inputPos = nullptr; // index of this node's entry in inputs[i]->users + U32* userSlots = nullptr; // slot of users[k] that points here U32 inputCount = 0; U32 inputCap = 0; U32 userCount = 0; diff --git a/src/backend/pass/verify.cpp b/src/backend/pass/verify.cpp index 97a7f22b..5841a46a 100644 --- a/src/backend/pass/verify.cpp +++ b/src/backend/pass/verify.cpp @@ -92,9 +92,8 @@ namespace rat { err(n, "input " + std::to_string(i) + " (" + vref(in) + ") is not a node of this function"); continue; } - if(!listsAsUser(in, n)) - err(n, - "broken reverse edge: input " + vref(in) + " does not list " + vref(n) + " as a user"); + if(!n->inputEdgeIndexed(i)) + err(n, "broken reverse edge: input " + vref(in) + " slot " + std::to_string(i)); } for(Node* u : n->getUsers()) { if(!u) { From 68457c23a6692fee651ab1c729039e1af2c9fcf8 Mon Sep 17 00:00:00 2001 From: nella Date: Mon, 12 Oct 2026 18:12:51 +0200 Subject: [PATCH 02/16] Do not reassociate wrapping constants. --- src/backend/pass/opt/fold.cpp | 10 ++++++-- src/backend/test/fold_reassoc_no_wrap.rat | 30 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 src/backend/test/fold_reassoc_no_wrap.rat diff --git a/src/backend/pass/opt/fold.cpp b/src/backend/pass/opt/fold.cpp index 9d9d1343..ab3db4fd 100644 --- a/src/backend/pass/opt/fold.cpp +++ b/src/backend/pass/opt/fold.cpp @@ -314,14 +314,20 @@ namespace rat { B32 innerNeg = inner == Opcode::Sub; U64 t1 = innerNeg ? (U64)0 - (U64)c1 : (U64)c1; U64 t2 = outerNeg ? (U64)0 - (U64)c2 : (U64)c2; - I64 k = normalizeConst((I64)(t1 + t2), w); + I64 sum = (I64)(t1 + t2); + I64 k = normalizeConst(sum, w); + if(w > 1 && w < 64 && k != sum) + return nullptr; if(k == 0) return base; return mkBin(fn, Opcode::Add, ty, base, k); // sub-by-const normalizes to add } } if(op == Opcode::Mul && matchVarConst(lhs, Opcode::Mul, base, c1)) { - I64 k = normalizeConst((I64)((U64)c1 * (U64)cr->getValue()), w); + I64 product = (I64)((U64)c1 * (U64)cr->getValue()); + I64 k = normalizeConst(product, w); + if(w > 1 && w < 64 && k != product) + return nullptr; if(k == 0) return constant(fn, ty, 0); if(k == 1) diff --git a/src/backend/test/fold_reassoc_no_wrap.rat b/src/backend/test/fold_reassoc_no_wrap.rat new file mode 100644 index 00000000..2f10af13 --- /dev/null +++ b/src/backend/test/fold_reassoc_no_wrap.rat @@ -0,0 +1,30 @@ +@name fold: (x + 2147483645) + 3 on i32 stays unfolded, the folded constant would wrap +@passes fold + +@input +func t(i32) -> i32 { + v0 = start : (ctrl, mem, i32) + v1 = stop : ctrl v9 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = const : i32 3 + v5 = const : i32 2147483645 + v6 = proj : i32 #2 "arg0" of v0 + v7 = add : i32 v6, v5 + v8 = add : i32 v7, v4 + v9 = return : ctrl v2, v3, v8 +} + +@expect +func t(i32) -> i32 { + v0 = start : (ctrl, mem, i32) + v1 = stop : ctrl v9 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = const : i32 3 + v5 = const : i32 2147483645 + v6 = proj : i32 #2 "arg0" of v0 + v7 = add : i32 v6, v5 + v8 = add : i32 v7, v4 + v9 = return : ctrl v2, v3, v8 +} From 8a1696911dc23ed42a1c16cd4721a18b7927341b Mon Sep 17 00:00:00 2001 From: nella Date: Mon, 12 Oct 2026 21:38:07 +0200 Subject: [PATCH 03/16] Keep foldable IV scales. --- src/backend/pass/opt/strength_reduce.cpp | 13 +++- src/backend/pass/opt/strength_reduce.h | 2 + src/backend/test/sr_keep_foldable_scale.rat | 67 +++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/backend/test/sr_keep_foldable_scale.rat diff --git a/src/backend/pass/opt/strength_reduce.cpp b/src/backend/pass/opt/strength_reduce.cpp index 02427da3..cabceb90 100644 --- a/src/backend/pass/opt/strength_reduce.cpp +++ b/src/backend/pass/opt/strength_reduce.cpp @@ -29,6 +29,14 @@ namespace rat { return false; } + B32 foldableScale(Opcode op, I64 k, U32 width) { + if(width != 64) + return false; + if(op == Opcode::Shl) + return k >= 1 && k <= 3; + return k == 2 || k == 4 || k == 8; + } + ConstantNode* affineScale(Node* u, PhiNode* p) { if(u->getInputCount() != 2) return nullptr; @@ -42,7 +50,10 @@ namespace rat { if(u->getInput(0) == p && u->getInput(1) == p) return nullptr; // p * p is quadratic Node* other = u->getInput(0) == p ? u->getInput(1) : u->getInput(0); - return dyn_cast(other); + ConstantNode* c = dyn_cast(other); + if(c && foldableScale(op, c->getValue(), p->getType()->getIntWidth())) + return nullptr; + return c; } } // namespace detail diff --git a/src/backend/pass/opt/strength_reduce.h b/src/backend/pass/opt/strength_reduce.h index 50e7d07a..6c43ff40 100644 --- a/src/backend/pass/opt/strength_reduce.h +++ b/src/backend/pass/opt/strength_reduce.h @@ -5,6 +5,7 @@ #include "core.h" +#include "ir/opcode.h" #include "pass/pass.h" namespace rat { @@ -14,6 +15,7 @@ namespace rat { namespace detail { B32 matchLinearIV(PhiNode* p, I64& step, U32& recIdx); + B32 foldableScale(Opcode op, I64 k, U32 width); ConstantNode* affineScale(Node* u, PhiNode* p); } // namespace detail diff --git a/src/backend/test/sr_keep_foldable_scale.rat b/src/backend/test/sr_keep_foldable_scale.rat new file mode 100644 index 00000000..9c089830 --- /dev/null +++ b/src/backend/test/sr_keep_foldable_scale.rat @@ -0,0 +1,67 @@ +@name strengthreduce: an i64 IV scaled by 4 stays for x86 addressing, a scale of 12 is reduced +@passes strengthreduce + +@input +var a : [4096 x i32] = "" + +func f() -> void { + v0 = start : (ctrl, mem) + v1 = stop : ctrl v24 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = const : i64 0 + v5 = region : ctrl loop v2, v10 + v6 = phi : i64 v5, v4, v23 + v7 = const : i64 1000 + v8 = slt : i1 v6, v7 + v9 = if : (ctrl, ctrl) v5, v8 + v10 = proj : ctrl #0 "then" of v9 + v11 = proj : ctrl #1 "else" of v9 + v12 = global : ptr "a" + v13 = const : i64 2 + v14 = shl : i64 v6, v13 + v15 = add : ptr v12, v14 + v16 = const : i64 12 + v17 = mul : i64 v6, v16 + v18 = add : ptr v12, v17 + v19 = phi : mem v5, v3, v21 + v20 = load : i32 v10, v19, v18 + v21 = store : mem v10, v19, v15, v20 + v22 = const : i64 1 + v23 = add : i64 v6, v22 + v24 = return : ctrl v11, v19 +} + +@expect +var a : [4096 x i32] = "" + +func f() -> void { + v0 = start : (ctrl, mem) + v1 = stop : ctrl v24 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = const : i64 0 + v5 = region : ctrl loop v2, v10 + v6 = phi : i64 v5, v4, v23 + v7 = const : i64 1000 + v8 = slt : i1 v6, v7 + v9 = if : (ctrl, ctrl) v5, v8 + v10 = proj : ctrl #0 "then" of v9 + v11 = proj : ctrl #1 "else" of v9 + v12 = global : ptr "a" + v13 = const : i64 2 + v14 = shl : i64 v6, v13 + v15 = add : ptr v12, v14 + v18 = add : ptr v12, v27 + v19 = phi : mem v5, v3, v21 + v20 = load : i32 v10, v19, v18 + v21 = store : mem v10, v19, v15, v20 + v22 = const : i64 1 + v23 = add : i64 v6, v22 + v24 = return : ctrl v11, v19 + v25 = const : i64 12 + v26 = mul : i64 v4, v25 + v27 = phi : i64 v5, v26, v29 + v28 = const : i64 12 + v29 = add : i64 v27, v28 +} From f6877096d7a089e9154708b0fd580797a6281505 Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 00:26:41 +0200 Subject: [PATCH 04/16] Widen i32 induction variables. --- src/backend/pass/opt/strength_reduce.cpp | 98 +++++++++++++++++++ src/backend/pass/opt/strength_reduce.h | 7 ++ src/backend/test/sr_no_widen_param_bound.rat | 58 +++++++++++ src/backend/test/sr_widen_i32_iv.rat | 72 ++++++++++++++ .../test/correctness/custom/iv_widen_bounds.c | 49 ++++++++++ 5 files changed, 284 insertions(+) create mode 100644 src/backend/test/sr_no_widen_param_bound.rat create mode 100644 src/backend/test/sr_widen_i32_iv.rat create mode 100644 src/compiler/test/correctness/custom/iv_widen_bounds.c diff --git a/src/backend/pass/opt/strength_reduce.cpp b/src/backend/pass/opt/strength_reduce.cpp index cabceb90..21a7af02 100644 --- a/src/backend/pass/opt/strength_reduce.cpp +++ b/src/backend/pass/opt/strength_reduce.cpp @@ -55,10 +55,104 @@ namespace rat { return nullptr; return c; } + + // p < c or p <= c for a constant c, whose then arm is the back edge of p + CompareNode* loopTest(PhiNode* p, U32 recIdx) { + ProjNode* thenP = dyn_cast(p->getRegion()->getPredecessor(recIdx)); + if(!thenP || thenP->getIndex() != IfNode::thenProjIndex()) + return nullptr; + IfNode* iff = dyn_cast(thenP->getProducer()); + if(!iff || iff->getControl() != p->getRegion()) + return nullptr; + CompareNode* cmp = dyn_cast(iff->getPredicate()); + if(!cmp || cmp->getLHS() != p || !isa(cmp->getRHS())) + return nullptr; + Opcode op = cmp->getOpcode(); + if(op != Opcode::Slt && op != Opcode::Sle) + return nullptr; + return cmp; + } + + // k of u = p + k (null otherwise) + ConstantNode* offsetOf(Node* u, PhiNode* p) { + if(u->getOpcode() != Opcode::Add) + return nullptr; + if(u->getInput(0) == p) + return dyn_cast(u->getInput(1)); + if(u->getInput(1) == p) + return dyn_cast(u->getInput(0)); + return nullptr; + } + + B32 isSext64(Node* u) { + return u->getOpcode() == Opcode::SExt && u->getType()->getIntWidth() == 64; + } + + B32 widenableUse(Node* u, PhiNode* p, I64 lo, I64 top) { + if(isSext64(u)) + return true; + ConstantNode* c = offsetOf(u, p); + if(!c) + return false; + I64 k = signExtend(c->getValue(), 32); + B32 fits = lo + k >= INT32_MIN && top + k <= INT32_MAX; + for(Node* s : u->getUsers()) + if(s != p && (!isSext64(s) || !fits)) + return false; + return true; + } + + // make u read q, the i64 IV of p + void widenUse(Function& fn, Node* u, PhiNode* p, PhiNode* q, CompareNode* cmp, I64 bound) { + Type* i64 = q->getType(); + if(u == cmp) { + Node* rhs = constant(fn, i64, bound); + cmp->replaceAllUsesWith(fn.create(cmp->getOpcode(), cmp->getType(), q, rhs)); + } else if(ConstantNode* c = offsetOf(u, p)) { + Node* k = constant(fn, i64, signExtend(c->getValue(), 32)); + Node* sum = fn.create(Opcode::Add, i64, q, k); + List sexts(u->getUsers().begin(), u->getUsers().end()); + for(Node* s : sexts) + if(s != p) + s->replaceAllUsesWith(sum); + } else { + u->replaceAllUsesWith(q); + } + } } // namespace detail const C8* StrengthReducePass::name() const { return "strengthreduce"; } + // turn an i32 IV read only by its loop test, sexts and p + k sums into an i64 IV + U32 StrengthReducePass::widen(Function& fn, PhiNode* p, I64 step, U32 recIdx) { + CompareNode* cmp = detail::loopTest(p, recIdx); + Node* init = p->getValue(1 - recIdx); + if(!cmp || !isa(init) || p->getType()->getIntWidth() != 32) + return 0; + I64 lo = signExtend(cast(init)->getValue(), 32); + I64 bound = signExtend(cast(cmp->getRHS())->getValue(), 32); + I64 limit = bound; + if(cmp->getOpcode() == Opcode::Sle) + limit = bound + 1; + step = signExtend(step, 32); + I64 top = std::max(lo, limit - 1 + step); + if(step <= 0 || top > INT32_MAX) + return 0; + for(Node* u : p->getUsers()) + if(u != cmp && !detail::widenableUse(u, p, lo, top)) + return 0; + + Type* i64 = fn.types().getInt(64); + Node* init64 = constant(fn, i64, lo); + PhiNode* q = fn.create(i64, List{p->getRegion(), init64, init64}); + q->setInput(1 + recIdx, fn.create(Opcode::Add, i64, q, constant(fn, i64, step))); + List users(p->getUsers().begin(), p->getUsers().end()); + for(Node* u : users) + detail::widenUse(fn, u, p, q, cmp, bound); + p->setInput(1 + recIdx, init); // p and its step are dead, cut their cycle + return 1; + } + U32 StrengthReducePass::runOnFunction(Function& fn, const TargetInfo&) { U32 changed = 0; @@ -72,6 +166,10 @@ namespace rat { U32 recIdx; if(!detail::matchLinearIV(p, step, recIdx)) continue; + if(U32 widened = widen(fn, p, step, recIdx)) { + changed += widened; + continue; + } Type* ty = p->getType(); U32 w = ty->getIntWidth(); Node* init = p->getValue(1 - recIdx); diff --git a/src/backend/pass/opt/strength_reduce.h b/src/backend/pass/opt/strength_reduce.h index 6c43ff40..8ab15a9e 100644 --- a/src/backend/pass/opt/strength_reduce.h +++ b/src/backend/pass/opt/strength_reduce.h @@ -9,6 +9,7 @@ #include "pass/pass.h" namespace rat { + struct CompareNode; struct ConstantNode; struct Node; struct PhiNode; @@ -17,11 +18,17 @@ namespace rat { B32 matchLinearIV(PhiNode* p, I64& step, U32& recIdx); B32 foldableScale(Opcode op, I64 k, U32 width); ConstantNode* affineScale(Node* u, PhiNode* p); + CompareNode* loopTest(PhiNode* p, U32 recIdx); + ConstantNode* offsetOf(Node* u, PhiNode* p); + B32 isSext64(Node* u); + B32 widenableUse(Node* u, PhiNode* p, I64 lo, I64 top); + void widenUse(Function& fn, Node* u, PhiNode* p, PhiNode* q, CompareNode* cmp, I64 bound); } // namespace detail struct StrengthReducePass : FunctionPass { const C8* name() const override; U32 runOnFunction(Function& fn, const TargetInfo& target) override; + static U32 widen(Function& fn, PhiNode* p, I64 step, U32 recIdx); }; } // namespace rat diff --git a/src/backend/test/sr_no_widen_param_bound.rat b/src/backend/test/sr_no_widen_param_bound.rat new file mode 100644 index 00000000..b21d4635 --- /dev/null +++ b/src/backend/test/sr_no_widen_param_bound.rat @@ -0,0 +1,58 @@ +@name strengthreduce: an i32 IV with a parameter bound is not widened (its step may wrap) +@passes strengthreduce + +@input +var a : [1024 x i32] = "" + +func f(i32) -> void { + v0 = start : (ctrl, mem, i32) + v1 = stop : ctrl v21 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = proj : i32 #2 "arg0" of v0 + v5 = const : i32 0 + v6 = region : ctrl loop v2, v10 + v7 = phi : i32 v6, v5, v20 + v8 = slt : i1 v7, v4 + v9 = if : (ctrl, ctrl) v6, v8 + v10 = proj : ctrl #0 "then" of v9 + v11 = proj : ctrl #1 "else" of v9 + v12 = global : ptr "a" + v13 = sext : i64 v7 + v14 = const : i64 2 + v15 = shl : i64 v13, v14 + v16 = add : ptr v12, v15 + v17 = phi : mem v6, v3, v18 + v18 = store : mem v10, v17, v16, v7 + v19 = const : i32 2 + v20 = add : i32 v7, v19 + v21 = return : ctrl v11, v17 +} + +@expect +var a : [1024 x i32] = "" + +func f(i32) -> void { + v0 = start : (ctrl, mem, i32) + v1 = stop : ctrl v21 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = proj : i32 #2 "arg0" of v0 + v5 = const : i32 0 + v6 = region : ctrl loop v2, v10 + v7 = phi : i32 v6, v5, v20 + v8 = slt : i1 v7, v4 + v9 = if : (ctrl, ctrl) v6, v8 + v10 = proj : ctrl #0 "then" of v9 + v11 = proj : ctrl #1 "else" of v9 + v12 = global : ptr "a" + v13 = sext : i64 v7 + v14 = const : i64 2 + v15 = shl : i64 v13, v14 + v16 = add : ptr v12, v15 + v17 = phi : mem v6, v3, v18 + v18 = store : mem v10, v17, v16, v7 + v19 = const : i32 2 + v20 = add : i32 v7, v19 + v21 = return : ctrl v11, v17 +} diff --git a/src/backend/test/sr_widen_i32_iv.rat b/src/backend/test/sr_widen_i32_iv.rat new file mode 100644 index 00000000..2ecdc39e --- /dev/null +++ b/src/backend/test/sr_widen_i32_iv.rat @@ -0,0 +1,72 @@ +@name strengthreduce: a counted i32 IV read through sext becomes an i64 IV +@passes strengthreduce + +@input +var a : [1024 x i32] = "" +var b : [1024 x i32] = "" + +func f() -> void { + v0 = start : (ctrl, mem) + v1 = stop : ctrl v29 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v4 = const : i32 0 + v5 = region : ctrl loop v2, v10 + v6 = phi : i32 v5, v4, v28 + v7 = const : i32 1000 + v8 = slt : i1 v6, v7 + v9 = if : (ctrl, ctrl) v5, v8 + v10 = proj : ctrl #0 "then" of v9 + v11 = proj : ctrl #1 "else" of v9 + v12 = global : ptr "a" + v13 = sext : i64 v6 + v14 = const : i64 2 + v15 = shl : i64 v13, v14 + v16 = add : ptr v12, v15 + v17 = global : ptr "b" + v18 = const : i32 1 + v19 = add : i32 v6, v18 + v20 = sext : i64 v19 + v21 = shl : i64 v20, v14 + v22 = add : ptr v17, v21 + v23 = phi : mem v5, v3, v25 + v24 = load : i32 v10, v23, v22 + v25 = store : mem v10, v23, v16, v24 + v26 = const : i32 2 + v28 = add : i32 v6, v26 + v29 = return : ctrl v11, v23 +} + +@expect +var a : [1024 x i32] = "" +var b : [1024 x i32] = "" + +func f() -> void { + v0 = start : (ctrl, mem) + v1 = stop : ctrl v28 + v2 = proj : ctrl #0 "ctrl" of v0 + v3 = proj : mem #1 "mem" of v0 + v5 = region : ctrl loop v2, v10 + v9 = if : (ctrl, ctrl) v5, v34 + v10 = proj : ctrl #0 "then" of v9 + v11 = proj : ctrl #1 "else" of v9 + v12 = global : ptr "a" + v14 = const : i64 2 + v15 = shl : i64 v30, v14 + v16 = add : ptr v12, v15 + v17 = global : ptr "b" + v21 = shl : i64 v36, v14 + v22 = add : ptr v17, v21 + v23 = phi : mem v5, v3, v25 + v24 = load : i32 v10, v23, v22 + v25 = store : mem v10, v23, v16, v24 + v28 = return : ctrl v11, v23 + v29 = const : i64 0 + v30 = phi : i64 v5, v29, v32 + v31 = const : i64 2 + v32 = add : i64 v30, v31 + v33 = const : i64 1000 + v34 = slt : i1 v30, v33 + v35 = const : i64 1 + v36 = add : i64 v30, v35 +} diff --git a/src/compiler/test/correctness/custom/iv_widen_bounds.c b/src/compiler/test/correctness/custom/iv_widen_bounds.c new file mode 100644 index 00000000..02037c71 --- /dev/null +++ b/src/compiler/test/correctness/custom/iv_widen_bounds.c @@ -0,0 +1,49 @@ +// expect: 0 +// counted int loops whose IV strengthreduce widens to 64 bits, next to ones it must leave alone + +int a[64]; + +__attribute__((noinline)) int sumStep(void) { // widened: 0 <= i < 64, step 4, i + 1..3 + int s = 0; + for(int i = 0; i < 64; i += 4) + s += a[i] + a[i + 1] * 2 + a[i + 2] * 3 + a[i + 3] * 4; + return s; +} + +__attribute__((noinline)) int sumLe(void) { // widened: i <= 62, i - 1 with i >= 1 + int s = 0; + for(int i = 1; i <= 62; i++) + s += a[i - 1] - a[i + 1]; + return s; +} + +__attribute__((noinline)) int nearMax(void) { // i stays below INT_MAX, but i + 3 at the exit value would not + int s = 0; + for(int i = 2147483647 - 40; i < 2147483647 - 3; i += 4) + s += a[(i + 3) - (2147483647 - 40)]; + return s; +} + +__attribute__((noinline)) int empty(void) { // the back edge never runs + int s = 0; + for(int i = 100; i < 64; i++) + s += a[i - 100]; + return s; +} + +int main(void) { + for(int i = 0; i < 64; i++) + a[i] = i * 7 + 1; + int ref = 0; + for(int i = 0; i < 64; i += 4) + ref += a[i] + a[i + 1] * 2 + a[i + 2] * 3 + a[i + 3] * 4; + if(sumStep() != ref) + return 1; + if(sumLe() != -14 * 62) + return 2; + if(nearMax() != a[3] + a[7] + a[11] + a[15] + a[19] + a[23] + a[27] + a[31] + a[35] + a[39]) + return 3; + if(empty() != 0) + return 4; + return 0; +} From c0339319b201d18e1f34593ab4e351adc9f54841 Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 10:54:18 +0200 Subject: [PATCH 05/16] Emit induction steps last. --- src/backend/codegen/schedule.cpp | 21 ++++++++++++++++++++- src/backend/codegen/schedule.h | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/backend/codegen/schedule.cpp b/src/backend/codegen/schedule.cpp index ffdc7016..481b8195 100644 --- a/src/backend/codegen/schedule.cpp +++ b/src/backend/codegen/schedule.cpp @@ -615,6 +615,24 @@ namespace rat { flat[fill[keys[i]]++] = items[i]; } + B32 Schedule::emitsEarly(const Node* n) { + NodeSpan users = n->getUsers(); + if(users.empty() || isa(n) || !isFloating(n)) + return true; + for(Node* u : users) + if(!isa(u)) + return true; + B32 readsUser = false; + for(U32 i = 0, e = n->getInputCount(); i < e; ++i) { + Node* in = n->getInput(i); + B32 user = std::find(users.begin(), users.end(), in) != users.end(); + if(!user && !isa(in)) + return true; + readsUser |= user; + } + return !readsUser; + } + void Schedule::buildBlockLists() { U32 nb = (U32)blocks.size(); List keys; @@ -622,7 +640,7 @@ namespace rat { keys.push_back(headBlock(cast(phi)->getRegion())); bucket(dataPhis, keys, nb, phiStart, phiFlat); - // placed nodes in function (id) order + // placed nodes in function (id) order, induction steps last List listed; for(Node* n : floating) if(nodeBlock[n->getId()] >= 0) @@ -630,6 +648,7 @@ namespace rat { auto mid = listed.insert(listed.end(), pinned.begin(), pinned.end()); std::sort(mid, listed.end(), detail::earlierId); std::inplace_merge(listed.begin(), mid, listed.end(), detail::earlierId); + std::stable_partition(listed.begin(), listed.end(), emitsEarly); keys.clear(); for(Node* n : listed) keys.push_back(nodeBlock[n->getId()]); diff --git a/src/backend/codegen/schedule.h b/src/backend/codegen/schedule.h index 947c676d..b7437baa 100644 --- a/src/backend/codegen/schedule.h +++ b/src/backend/codegen/schedule.h @@ -95,6 +95,7 @@ namespace rat { void placeLoads(const List& loads, const List& early); B32 place(Node* n, I32 late, const List& early); void buildBlockLists(); + static B32 emitsEarly(const Node* n); static void bucket(const List& items, const List& keys, U32 nb, @@ -121,7 +122,7 @@ namespace rat { List succHead; // per local index: head of the extra-edge chain (-1) List succNext; // edge -> next edge in the chain List succTo; // edge -> target local index - List ready; // binary heap of ready local indices, local order is id order + List ready; // binary heap of ready local indices, local order is list order List out; // topological order when some edge runs backward B32 backward = false; // some edge runs against local order From 5e7e9e8e31f5bd52a85f637c2765d4849e1cb94c Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 12:31:09 +0200 Subject: [PATCH 06/16] Speculate loads of defined globals. --- src/backend/codegen/schedule.cpp | 15 ++++++++++++++- src/backend/codegen/schedule.h | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/backend/codegen/schedule.cpp b/src/backend/codegen/schedule.cpp index 481b8195..8245408e 100644 --- a/src/backend/codegen/schedule.cpp +++ b/src/backend/codegen/schedule.cpp @@ -2,6 +2,7 @@ #include "analysis/alias_analysis.h" #include "ir/function.h" +#include "ir/module.h" #include "ir/node.h" namespace rat { @@ -394,6 +395,18 @@ namespace rat { return d == 0 || (isSigned && d == -1); // INT_MIN / -1 traps } + // a load inside a global this module defines cannot fault, so it may run before a branch + B32 Schedule::loadsGlobal(const Node* n) const { + const LoadNode* l = dyn_cast(n); + if(!l) + return false; + const GlobalNode* g = dyn_cast(l->getPointer()); + if(!g) + return false; + const Global* obj = fn.getModule().getGlobal(g->getSymbol()); + return obj && !obj->isAlias() && obj->getType()->byteSize(8) >= l->getType()->byteSize(8); + } + B32 Schedule::isGuarded(const Node* n) const { if(isa(n)) return false; @@ -414,7 +427,7 @@ namespace rat { return late; // nothing above is shallower, so the walk cannot move it Opcode op = n->getOpcode(); B32 remat = op == Opcode::Constant || op == Opcode::Global; - B32 trapping = mayTrap(n) || (!guarded.empty() && guarded[n->getId()]); + B32 trapping = (mayTrap(n) && !loadsGlobal(n)) || (!guarded.empty() && guarded[n->getId()]); I32 cur = late, pick = late; while(true) { if(blocks[cur].loopDepth < blocks[pick].loopDepth) { diff --git a/src/backend/codegen/schedule.h b/src/backend/codegen/schedule.h index b7437baa..cb7ba8ae 100644 --- a/src/backend/codegen/schedule.h +++ b/src/backend/codegen/schedule.h @@ -112,6 +112,7 @@ namespace rat { I32 predBlockForRegionInput(I32 regionBlock, U32 i) const; I32 hoistTarget(const Node* n, I32 late, I32 early) const; I32 homeBlock(Node* n) const; + B32 loadsGlobal(const Node* n) const; B32 isGuarded(const Node* n) const; struct TopoScratch { From 8ad66cc9c0678e37474d16fa598e08fa7051fdcf Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 14:09:52 +0200 Subject: [PATCH 07/16] Find load clobbers by object. --- src/backend/analysis/alias_analysis.cpp | 10 +++ src/backend/analysis/alias_analysis.h | 2 + src/backend/codegen/schedule.cpp | 97 ++++++++++++++++++++++--- src/backend/codegen/schedule.h | 11 +++ 4 files changed, 109 insertions(+), 11 deletions(-) diff --git a/src/backend/analysis/alias_analysis.cpp b/src/backend/analysis/alias_analysis.cpp index abeed626..a905475f 100644 --- a/src/backend/analysis/alias_analysis.cpp +++ b/src/backend/analysis/alias_analysis.cpp @@ -80,6 +80,16 @@ namespace rat { return cast(a)->getSymbol() != cast(b)->getSymbol(); } + Node* AliasAnalysis::objectOf(Node* addr) const { + Node* base = decompose(addr).base; + if(!isIdentified(base)) + return nullptr; + GlobalNode* g = dyn_cast(base); + if(!g) + return base; + return symbolObjects.try_emplace(g->getSymbol(), base).first->second; + } + AliasResult AliasAnalysis::alias(Node* addrA, U32 sizeA, Node* addrB, U32 sizeB) const { // alias query between two addresses with known access sizes (0 = unknown) const Address& a = decompose(addrA); diff --git a/src/backend/analysis/alias_analysis.h b/src/backend/analysis/alias_analysis.h index d11eaa20..22a72f12 100644 --- a/src/backend/analysis/alias_analysis.h +++ b/src/backend/analysis/alias_analysis.h @@ -57,6 +57,7 @@ namespace rat { // object identity, independent of any offset static B32 isIdentified(const Node* n); // an Alloc or a Global static B32 distinctObjects(const Node* a, const Node* b); + Node* objectOf(Node* addr) const; private: struct Address { Node* base; // pointer @@ -69,6 +70,7 @@ namespace rat { private: U32 ptrBytes; mutable Map decomposeCache; + mutable Map symbolObjects; // first global node seen per symbol }; } // namespace rat diff --git a/src/backend/codegen/schedule.cpp b/src/backend/codegen/schedule.cpp index 8245408e..6f1c2345 100644 --- a/src/backend/codegen/schedule.cpp +++ b/src/backend/codegen/schedule.cpp @@ -702,6 +702,14 @@ namespace rat { } B32 earlierId(const Node* a, const Node* b) { return a->getId() < b->getId(); } + + // the object a store writes, null for other writers + Node* storedObject(const AliasAnalysis& aa, Node* w) { + StoreNode* st = dyn_cast(w); + if(!st) + return nullptr; + return aa.objectOf(st->getPointer()); + } } // namespace detail I32 Schedule::TopoScratch::local(const Node* n) const { return n ? localOf[n->getId()] : -1; } @@ -775,24 +783,91 @@ namespace rat { std::copy(s.out.begin(), s.out.end(), nodes); } + // a load runs before the first writer on its state's chain that may clobber it void Schedule::addAntiDeps(NodeSpan nodes, const AliasAnalysis& aa, TopoScratch& s) { + s.chain.clear(); + for(Node* n : nodes) { + LoadNode* ld = dyn_cast(n); + if(!ld || !ld->getMemory()) + continue; + I32 start = s.stHead[ld->getMemory()->getId()]; + if(start < 0) + continue; + if(I32 w = firstClobber(nodes, ld, start, aa, s); w >= 0) + s.addEdge(ld, nodes[w]); + } + } + + void Schedule::rankWriters(NodeSpan nodes, TopoScratch& s) { U32 k = (U32)nodes.size(); + s.rank.assign(k, -1); for(U32 i = 0; i < k; ++i) { - LoadNode* ld = dyn_cast(nodes[i]); - if(!ld) + Node* mem = memoryInputOf(nodes[i]); + if(!mem || !isMemWriter(nodes[i])) continue; - Node* state = ld->getMemory(); - U32 guard = 0; // chain length bound; conservatively pin if exceeded - for(I32 wi = state ? s.stHead[state->getId()] : -1; wi >= 0; ++guard) { - Node* w = nodes[wi]; - StoreNode* st = dyn_cast(w); - if(!st || detail::storeMayAliasLoad(aa, st, ld) || guard >= k) { - s.addEdge(ld, w); // call, aliasing store, or bound hit -> pin here + if(isa(mem) && s.local(mem) >= 0 && s.stHead[mem->getId()] == (I32)i) + continue; // inside the chain of mem + for(I32 w = (I32)i; w >= 0; w = s.stHead[nodes[w]->getId()]) { + s.rank[w] = (I32)s.chain.size(); + s.chain.push_back(w); + if(!isa(nodes[w])) break; - } - wi = s.stHead[w->getId()]; // skip disjoint store, walk to next writer } + s.chain.push_back(-1); + } + } + + void Schedule::indexObjects(NodeSpan nodes, const AliasAnalysis& aa, TopoScratch& s) { + s.nextWild.resize(s.chain.size()); + s.byObject.clear(); + for(U32 p = (U32)s.chain.size(); p-- > 0;) { + I32 w = s.chain[p]; + Node* obj = nullptr; + if(w >= 0) + obj = detail::storedObject(aa, nodes[w]); + if(obj) { + s.nextWild[p] = s.nextWild[p + 1]; + s.byObject.push_back({obj->getId(), (I32)p}); + } else { + s.nextWild[p] = (I32)p; + } + } + std::sort(s.byObject.begin(), s.byObject.end()); + } + + // first writer from start on that is not a store or may alias ld, or -1 + I32 Schedule::walkClobber( + NodeSpan nodes, LoadNode* ld, I32 start, const AliasAnalysis& aa, const TopoScratch& s) { + I32 w = start; + while(w >= 0) { + StoreNode* st = dyn_cast(nodes[w]); + if(!st || detail::storeMayAliasLoad(aa, st, ld)) + break; + w = s.stHead[st->getId()]; + } + return w; + } + + I32 Schedule::firstClobber( + NodeSpan nodes, LoadNode* ld, I32 start, const AliasAnalysis& aa, TopoScratch& s) { + Node* obj = aa.objectOf(ld->getPointer()); + if(!obj) + return walkClobber(nodes, ld, start, aa, s); // a store to another base may alias + if(s.chain.empty()) { + rankWriters(nodes, s); + indexObjects(nodes, aa, s); + } + I32 r = s.rank[start]; + assert(r >= 0 && "writer outside every chain"); + I32 bound = s.nextWild[r]; + Pair key{obj->getId(), r}; + auto it = std::lower_bound(s.byObject.begin(), s.byObject.end(), key); + for(; it != s.byObject.end() && it->first == key.first && it->second < bound; ++it) { + I32 w = s.chain[it->second]; + if(detail::storeMayAliasLoad(aa, cast(nodes[w]), ld)) + return w; } + return s.chain[bound]; } void Schedule::addOrderEdges(NodeSpan nodes, TopoScratch& s) { diff --git a/src/backend/codegen/schedule.h b/src/backend/codegen/schedule.h index cb7ba8ae..dc00aa01 100644 --- a/src/backend/codegen/schedule.h +++ b/src/backend/codegen/schedule.h @@ -20,6 +20,7 @@ namespace rat { namespace detail { B32 storeMayAliasLoad(const AliasAnalysis& aa, const StoreNode* st, const LoadNode* ld); B32 earlierId(const Node* a, const Node* b); + Node* storedObject(const AliasAnalysis& aa, Node* w); } // namespace detail struct Schedule { @@ -126,6 +127,10 @@ namespace rat { List ready; // binary heap of ready local indices, local order is list order List out; // topological order when some edge runs backward B32 backward = false; // some edge runs against local order + List chain; // writer local indices in memory order, each chain ends in -1 + List rank; // per local index: position of the writer in chain (-1) + List nextWild; + List> byObject; // (object id, chain position) of stores to objects, sorted I32 local(const Node* n) const; void addEdge(Node* before, Node* after); @@ -134,6 +139,12 @@ namespace rat { }; void topoOrder(Node** nodes, U32 k, const AliasAnalysis& aa, TopoScratch& scratch) const; static void addAntiDeps(NodeSpan nodes, const AliasAnalysis& aa, TopoScratch& s); + static void rankWriters(NodeSpan nodes, TopoScratch& s); + static void indexObjects(NodeSpan nodes, const AliasAnalysis& aa, TopoScratch& s); + static I32 walkClobber( + NodeSpan nodes, LoadNode* ld, I32 start, const AliasAnalysis& aa, const TopoScratch& s); + static I32 + firstClobber(NodeSpan nodes, LoadNode* ld, I32 start, const AliasAnalysis& aa, TopoScratch& s); static void addOrderEdges(NodeSpan nodes, TopoScratch& s); I32 fixedDataBlock(Node* n, const List& early) const; From f2dc7ba7a282bb95b4fabffd17a6e8a9e47dc7bf Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 15:47:33 +0200 Subject: [PATCH 08/16] Track restrict parameters. --- src/backend/ir/function.h | 3 +++ src/backend/ir/text_parser.cpp | 23 ++++++++++++++++++----- src/backend/ir/text_parser.h | 5 +++-- src/backend/pass/emit/text_emitter.cpp | 2 ++ src/compiler/emit/emit.cpp | 19 ++++++++++++++++++- src/compiler/emit/emit.h | 2 ++ src/compiler/emit/emit_state.cpp | 1 + src/compiler/emit/emit_state.h | 1 + src/compiler/parse/ast.h | 3 +++ src/compiler/parse/parse_declarator.cpp | 7 ++++++- 10 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/backend/ir/function.h b/src/backend/ir/function.h index 153e1cf7..8a7e0ae7 100644 --- a/src/backend/ir/function.h +++ b/src/backend/ir/function.h @@ -19,11 +19,14 @@ namespace rat { // symbol linkage, shared by functions and globals enum class Linkage { External, Internal }; + constexpr U32 kMaxRestrictParams = 64; + struct FunctionAttrs { Linkage linkage = Linkage::External; U32 align = 0; // 0 = target default B32 variadic = false; B32 noInline = false; + U64 restrictParams = 0; B32 isInternal() const { return linkage == Linkage::Internal; } }; diff --git a/src/backend/ir/text_parser.cpp b/src/backend/ir/text_parser.cpp index 4466736b..920fd491 100644 --- a/src/backend/ir/text_parser.cpp +++ b/src/backend/ir/text_parser.cpp @@ -222,7 +222,7 @@ namespace rat { return elem; } - B32 Parser::parseTypeList(const String& s, List& out) { + B32 Parser::parseTypeList(const String& s, List& out, U64* restricts) { U32 depth = 0; String cur; for(C8 c : s) { @@ -231,20 +231,22 @@ namespace rat { if(c == ')') --depth; if(c == ',' && depth == 0) { - if(!flushType(cur, out)) + if(!flushType(cur, out, restricts)) return false; } else { cur.push_back(c); } } - return flushType(cur, out); + return flushType(cur, out, restricts); } - B32 Parser::flushType(String& cur, List& out) { + B32 Parser::flushType(String& cur, List& out, U64* restricts) { String e = trim(cur); cur.clear(); if(e.empty()) return true; + if(restricts && takeSuffix(e, " restrict") && out.size() < kMaxRestrictParams) + *restricts |= (U64)1 << out.size(); Type* et = parseType(e); if(!et) return false; @@ -294,7 +296,8 @@ namespace rat { } List params; - if(!parseTypeList(paramsStr, params)) + U64 restricts = 0; + if(!parseTypeList(paramsStr, params, &restricts)) return false; Type* ret = nullptr; @@ -306,6 +309,7 @@ namespace rat { Function* fn = mod.createFunction(name, params, ret); fn->getAttrs().noInline = noInline; + fn->getAttrs().restrictParams = restricts; List parsed; String line; @@ -435,6 +439,15 @@ namespace rat { return true; } + // strip a trailing word that follows other text, then trim the rest + B32 Parser::takeSuffix(String& body, const C8* word) { + U32 n = (U32)std::strlen(word); + if(body.size() <= n || body.compare(body.size() - n, n, word) != 0) + return false; + body = trim(body.substr(0, body.size() - n)); + return true; + } + B32 Parser::parseProj(const String& remainder, const String& line, ParsedNode& pn) { std::istringstream ss(remainder); String index; diff --git a/src/backend/ir/text_parser.h b/src/backend/ir/text_parser.h index b1ae4f83..8bd47f3a 100644 --- a/src/backend/ir/text_parser.h +++ b/src/backend/ir/text_parser.h @@ -52,8 +52,8 @@ namespace rat { B32 parse(std::istream& in); Type* parseType(const String& s); Type* parseCounted(const String& t, const C8* kind, const C8* countName, U32& count); - B32 parseTypeList(const String& s, List& out); - B32 flushType(String& cur, List& out); + B32 parseTypeList(const String& s, List& out, U64* restricts = nullptr); + B32 flushType(String& cur, List& out, U64* restricts); B32 parseGlobal(const String& line); B32 parseFunction(const String& header, std::istream& in); B32 parseNodeLine(const String& line, ParsedNode& pn); @@ -62,6 +62,7 @@ namespace rat { B32 parseLaneOp(const String& remainder, const String& line, ParsedNode& pn); B32 singleRef(const String& remainder, const String& error, ParsedNode& pn); static B32 takeKeyword(String& body, const C8* word); + static B32 takeSuffix(String& body, const C8* word); B32 build(Function* fn, const List& nodes); void seedStartStop(Function* fn, const List& nodes); diff --git a/src/backend/pass/emit/text_emitter.cpp b/src/backend/pass/emit/text_emitter.cpp index 8afe32fd..b55c6af0 100644 --- a/src/backend/pass/emit/text_emitter.cpp +++ b/src/backend/pass/emit/text_emitter.cpp @@ -110,6 +110,8 @@ namespace rat { if(i) *os << ", "; fn.getParamType(i)->print(*os); + if(i < kMaxRestrictParams && (fn.getAttrs().restrictParams >> i & 1)) + *os << " restrict"; } *os << ") -> "; if(fn.returnsValue()) diff --git a/src/compiler/emit/emit.cpp b/src/compiler/emit/emit.cpp index 42ed4e43..a8c2a57b 100644 --- a/src/compiler/emit/emit.cpp +++ b/src/compiler/emit/emit.cpp @@ -342,6 +342,7 @@ namespace rat::cc { void Emitter::storeLValue(Function& fn, const LValue& lv, Node* value) { if(lv.isVar()) { fn.set(lv.var, value); + dropRestrictParam(fn, lv.var); return; } B32 isVolatile = isTopVolatile(lv.type); @@ -457,11 +458,27 @@ namespace rat::cc { fn.store(slot, arg, isTopVolatile(p.type)); func.scopes.declare(*p.name, Local::mem(slot, p.type)); } else { - func.scopes.declare(*p.name, Local::inVar(fn.declareLocal(*p.name, arg), p.type)); + Function::Var v = fn.declareLocal(*p.name, arg); + func.scopes.declare(*p.name, Local::inVar(v, p.type)); + if(isPointer(p.type) && isTopRestrict(p.type)) + markRestrictParam(fn, v, paramBase + i); } } } + void Emitter::markRestrictParam(Function& fn, Function::Var v, U32 index) { + if(index >= kMaxRestrictParams) + return; + fn.getAttrs().restrictParams |= (U64)1 << index; + func.restrictVars.push_back({v, index}); + } + + void Emitter::dropRestrictParam(Function& fn, Function::Var v) { + for(const auto& [var, index] : func.restrictVars) + if(var == v) + fn.getAttrs().restrictParams &= ~((U64)1 << index); + } + B32 Emitter::emitFunctionBody(const FuncDef* def) { diag.function = def->name; B32 sretReturn = isAggregate(def->retType); diff --git a/src/compiler/emit/emit.h b/src/compiler/emit/emit.h index 758f8422..3c67bf0d 100644 --- a/src/compiler/emit/emit.h +++ b/src/compiler/emit/emit.h @@ -129,6 +129,8 @@ namespace rat::cc { // functions B32 emitFunctionBody(const FuncDef* def); void bindFunctionParams(Function& fn, const FuncDef* def, U32 paramBase); + void markRestrictParam(Function& fn, Function::Var v, U32 index); + void dropRestrictParam(Function& fn, Function::Var v); B32 checkAliases(); // types and sizes diff --git a/src/compiler/emit/emit_state.cpp b/src/compiler/emit/emit_state.cpp index de77cc4c..6c35ad49 100644 --- a/src/compiler/emit/emit_state.cpp +++ b/src/compiler/emit/emit_state.cpp @@ -47,6 +47,7 @@ namespace rat::cc { sp = nullptr; sawAlloca = false; addrTaken.clear(); + restrictVars.clear(); labels.clear(); loops.clear(); switches.clear(); diff --git a/src/compiler/emit/emit_state.h b/src/compiler/emit/emit_state.h index bb25604a..833cd7e2 100644 --- a/src/compiler/emit/emit_state.h +++ b/src/compiler/emit/emit_state.h @@ -81,6 +81,7 @@ namespace rat::cc { Node* sp = nullptr; B32 sawAlloca = false; Set addrTaken; + List> restrictVars; Map labels; List loops; List> switches; diff --git a/src/compiler/parse/ast.h b/src/compiler/parse/ast.h index 3b7a9e3e..129bd972 100644 --- a/src/compiler/parse/ast.h +++ b/src/compiler/parse/ast.h @@ -85,6 +85,7 @@ namespace rat::cc { U32 ptr = 0; U32 quals = 0; U32 vquals = 0; + U32 rquals = 0; Base base = Base::Int; U8 mods = 0; const StructType* strukt = nullptr; @@ -113,6 +114,8 @@ namespace rat::cc { constexpr B32 isTopVolatile(CType t) { return (t.vquals & (1u << t.ptr)) != 0; } constexpr void setTopVolatile(CType& t) { t.vquals |= (1u << t.ptr); } constexpr void clearTopVolatile(CType& t) { t.vquals &= ~(1u << t.ptr); } + constexpr B32 isTopRestrict(CType t) { return (t.rquals & (1u << t.ptr)) != 0; } + constexpr void setTopRestrict(CType& t) { t.rquals |= (1u << t.ptr); } constexpr B32 isFloating(CType t) { return t.isFloat() && !t.isComplex() && t.ptr == 0; } constexpr B32 isComplexType(CType t) { return t.isComplex() && t.ptr == 0; } constexpr CType complexElem(CType t) { diff --git a/src/compiler/parse/parse_declarator.cpp b/src/compiler/parse/parse_declarator.cpp index 6258d34a..b1779309 100644 --- a/src/compiler/parse/parse_declarator.cpp +++ b/src/compiler/parse/parse_declarator.cpp @@ -4,22 +4,27 @@ namespace rat::cc { // [ [qualifier]... * ]... [qualifier]... - // const and volatile are kept, one bit per pointer level + // const, volatile and restrict are kept, one bit per pointer level void Parser::parsePointers(CType& t) { for(;;) { B32 sawConst = false; B32 sawVolatile = false; + B32 sawRestrict = false; while(detail::isTypeQualifier(peek().kind)) { if(check(TokKind::KwConst)) sawConst = true; if(check(TokKind::KwVolatile)) sawVolatile = true; + if(check(TokKind::KwRestrict)) + sawRestrict = true; advance(); } if(sawConst && t.ptr < 32) setTopConst(t); if(sawVolatile && t.ptr < 32) setTopVolatile(t); + if(sawRestrict && t.ptr < 32) + setTopRestrict(t); if(!accept(TokKind::Star)) break; ++t.ptr; From 0529a69dc5c77db9240ee2464857d9b6c0724207 Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 17:23:04 +0200 Subject: [PATCH 09/16] Use restrict in alias analysis. --- src/backend/analysis/alias_analysis.cpp | 78 +++++++++++++++- src/backend/analysis/alias_analysis.h | 12 ++- src/backend/codegen/schedule.cpp | 2 +- src/backend/pass/opt/memory_opt.cpp | 2 +- src/backend/pass/opt/slp/slp_driver.cpp | 2 +- .../custom/restrict_based_on_diff.c | 90 +++++++++++++++++++ 6 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 src/compiler/test/correctness/custom/restrict_based_on_diff.c diff --git a/src/backend/analysis/alias_analysis.cpp b/src/backend/analysis/alias_analysis.cpp index a905475f..404fcbf1 100644 --- a/src/backend/analysis/alias_analysis.cpp +++ b/src/backend/analysis/alias_analysis.cpp @@ -1,6 +1,8 @@ #include "analysis/alias_analysis.h" +#include "ir/function.h" #include "ir/node.h" +#include "ir/opcode.h" #include "ir/type.h" namespace rat { @@ -8,8 +10,9 @@ namespace rat { B32 idLess(const Node* a, const Node* b) { return a->getId() < b->getId(); } } // namespace detail - AliasAnalysis::AliasAnalysis(U32 pointerBytes) - : ptrBytes(pointerBytes) {} + AliasAnalysis::AliasAnalysis(U32 pointerBytes, U64 restrictParams) + : ptrBytes(pointerBytes), + restrictParams(restrictParams) {} const AliasAnalysis::Address& AliasAnalysis::decompose(Node* addr) const { auto it = decomposeCache.find(addr); @@ -80,10 +83,77 @@ namespace rat { return cast(a)->getSymbol() != cast(b)->getSymbol(); } + I32 AliasAnalysis::paramIndex(const Node* n) { + const ProjNode* p = dyn_cast(n); + U32 first = StartNode::paramProjIndex(0); + if(!p || !isa(p->getProducer()) || p->getIndex() < first) + return -1; + return (I32)(p->getIndex() - first); + } + + B32 AliasAnalysis::isRestrictParam(I32 index, U64 restricts) { + return index >= 0 && index < (I32)kMaxRestrictParams && (restricts >> index & 1); + } + + B32 AliasAnalysis::mayCarryRestrict(List work, U64 restricts, U32 cap) { + Set seen; + while(!work.empty()) { + const Node* c = work.back(); + work.pop_back(); + if(!c || !seen.insert(c).second) + continue; + if(seen.size() > cap) + return true; + Opcode op = c->getOpcode(); + I32 index = paramIndex(c); + if(isRestrictParam(index, restricts)) + return true; + if(index >= 0 || op == Opcode::Constant || op == Opcode::Load || op == Opcode::Proj || + isIdentified(c)) + continue; + B32 pure = isArithmeticOpcode(op) || isVectorUtilOpcode(op) || op == Opcode::Select || + op == Opcode::Phi; + if(!pure) + return true; + for(U32 i = 0; i < c->getInputCount(); ++i) + if(const Node* in = c->getInput(i); in && in->getType() && in->getType()->isData()) + work.push_back(in); + } + return false; + } + + U64 AliasAnalysis::restrictFacts(const Function& fn) { + U64 restricts = fn.getAttrs().restrictParams; + if(!restricts) + return 0; + List taken; + for(Node* n : fn) { + if(StoreNode* s = dyn_cast(n); s && s->getInputCount()) + taken.push_back(s->getValue()); + else if(isa(n) || isa(n)) + for(U32 i = 2; i < n->getInputCount(); ++i) + taken.push_back(n->getInput(i)); + } + if(mayCarryRestrict(std::move(taken), restricts, ~0u)) + return 0; + return restricts; + } + Node* AliasAnalysis::objectOf(Node* addr) const { - Node* base = decompose(addr).base; - if(!isIdentified(base)) + const Address& a = decompose(addr); + Node* base = a.base; + I32 index = paramIndex(base); + B32 isRestrict = isRestrictParam(index, restrictParams); + if(!isRestrict && !isIdentified(base)) return nullptr; + for(Node* t : a.symbolic) + if(restrictParams && mayCarryRestrict({t}, restrictParams, kMaxCarryCone)) + return nullptr; + if(isRestrict) { + if(!paramObjects[index]) + paramObjects[index] = base; + return paramObjects[index]; + } GlobalNode* g = dyn_cast(base); if(!g) return base; diff --git a/src/backend/analysis/alias_analysis.h b/src/backend/analysis/alias_analysis.h index 22a72f12..76aa1b2d 100644 --- a/src/backend/analysis/alias_analysis.h +++ b/src/backend/analysis/alias_analysis.h @@ -11,6 +11,7 @@ #include "core.h" #include "hash.h" +#include "ir/function.h" namespace rat { struct Node; @@ -22,7 +23,9 @@ namespace rat { enum class AliasResult { NoAlias, MayAlias, MustAlias }; struct AliasAnalysis { - AliasAnalysis(U32 pointerBytes); + static constexpr U32 kMaxCarryCone = 32; // cone nodes mayCarryRestrict walks per address term + + AliasAnalysis(U32 pointerBytes, U64 restrictParams); AliasResult alias(Node* addrA, U32 sizeA, Node* addrB, U32 sizeB) const; @@ -57,6 +60,10 @@ namespace rat { // object identity, independent of any offset static B32 isIdentified(const Node* n); // an Alloc or a Global static B32 distinctObjects(const Node* a, const Node* b); + static I32 paramIndex(const Node* n); // parameter of a Start projection, else -1 + static B32 isRestrictParam(I32 index, U64 restricts); + static B32 mayCarryRestrict(List work, U64 restricts, U32 cap); + static U64 restrictFacts(const Function& fn); Node* objectOf(Node* addr) const; private: struct Address { @@ -69,8 +76,11 @@ namespace rat { static Node* accessAddress(Node* n); // pointer operand of a Load/Store (else null) private: U32 ptrBytes; + U64 restrictParams; mutable Map decomposeCache; mutable Map symbolObjects; // first global node seen per symbol + // first projection seen per restrict parameter + mutable Node* paramObjects[kMaxRestrictParams] = {}; }; } // namespace rat diff --git a/src/backend/codegen/schedule.cpp b/src/backend/codegen/schedule.cpp index 6f1c2345..7f39f048 100644 --- a/src/backend/codegen/schedule.cpp +++ b/src/backend/codegen/schedule.cpp @@ -670,7 +670,7 @@ namespace rat { TopoScratch scratch; scratch.localOf.assign(fn.idBound(), -1); scratch.stHead.assign(fn.idBound(), -1); - AliasAnalysis aa(8); + AliasAnalysis aa(8, AliasAnalysis::restrictFacts(fn)); for(U32 b = 0; b < nb; ++b) topoOrder( nodeFlat.data() + nodeStart[b], (U32)(nodeStart[b + 1] - nodeStart[b]), aa, scratch); diff --git a/src/backend/pass/opt/memory_opt.cpp b/src/backend/pass/opt/memory_opt.cpp index 4b070925..65b86d49 100644 --- a/src/backend/pass/opt/memory_opt.cpp +++ b/src/backend/pass/opt/memory_opt.cpp @@ -180,7 +180,7 @@ namespace rat { U32 MemoryOptPass::runOnFunction(Function& fn, const TargetInfo& target) { fn.eliminateDeadNodes(); - AliasAnalysis aa(target.getPointerSizeInBytes()); + AliasAnalysis aa(target.getPointerSizeInBytes(), 0); loads.clear(); for(Node* n : fn) diff --git a/src/backend/pass/opt/slp/slp_driver.cpp b/src/backend/pass/opt/slp/slp_driver.cpp index 4c7c44c4..f384f4e3 100644 --- a/src/backend/pass/opt/slp/slp_driver.cpp +++ b/src/backend/pass/opt/slp/slp_driver.cpp @@ -93,7 +93,7 @@ namespace rat { : fn(fn), ptrBytes(target.getPointerSizeInBytes()), sse41(target.hasSse41()), - aa(ptrBytes), + aa(ptrBytes, 0), stats(stats) {} const Slp::StoreAddr& slp::Slp::storeAddr(StoreNode* s) { diff --git a/src/compiler/test/correctness/custom/restrict_based_on_diff.c b/src/compiler/test/correctness/custom/restrict_based_on_diff.c new file mode 100644 index 00000000..1eec5e31 --- /dev/null +++ b/src/compiler/test/correctness/custom/restrict_based_on_diff.c @@ -0,0 +1,90 @@ +// expect: 0 +// a pointer whose offset is computed from another pointer is based on that one (C11 6.7.3.1p3): +// a + (d - a) is d, so its accesses may alias the ones through restrict d, also when the +// difference goes through memory or a callee, and d + (e - d) is e +int G[8]; + +// the load through x stays below the store through d +__attribute__((noinline)) void load(int* restrict d, int* a) { + int* x = a + (d - a); + d[0] = 1; + d[1] = x[0]; +} + +// a packed run on d whose lanes read through x +__attribute__((noinline)) void run(int* restrict d, int* a) { + int* x = a + (d - a); + d[1] = x[0]; + d[2] = x[1]; + d[3] = x[2]; + d[4] = x[3]; +} + +// y is a, not d: its load stays below the store through a +__attribute__((noinline)) void reverse(int* restrict d, int* a) { + int* y = d + (a - d); + a[0] = 1; + d[0] = y[0]; +} + +// y is e: the scheduler keeps its load above the store through e +__attribute__((noinline)) int sched(int* restrict d, int* restrict e) { + int* y = d + (e - d); + int v = y[0]; + e[0] = 1; + return v * 3 + e[1]; +} + +// an object plus a difference to d is d +__attribute__((noinline)) void object(int* restrict d) { + int* x = G + (d - G); + d[0] = 1; + d[1] = x[0]; +} + +// the difference read back from memory +__attribute__((noinline)) void memory(int* restrict d, int* a, long* t) { + *t = d - a; + d[0] = 1; + int* x = a + *t; + d[1] = x[0]; +} + +__attribute__((noinline)) void diff(long* t, int* d, int* a) { + *t = d - a; +} + +// the difference stored by a callee that takes d +__attribute__((noinline)) void call(int* restrict d, int* a, long* t) { + diff(t, d, a); + d[0] = 1; + int* x = a + *t; + d[1] = x[0]; +} + +int main(void) { + int bad = 0; + int b[8] = {7, 7, 7, 7, 7, 7, 7, 7}; + load(b, b + 2); + bad += b[1] != 1; + int c[8] = {1, 2, 3, 4, 5, 6, 7, 8}; + run(c, c + 3); + bad += c[1] != 1 || c[2] != 1 || c[3] != 1 || c[4] != 1; + int e[4] = {7, 7, 7, 7}; + reverse(e + 2, e); + bad += e[2] != 1; + int s[4] = {7, 7, 7, 7}; + bad += sched(s, s + 2) != 28; + for(int i = 0; i < 8; i++) + G[i] = 7; + object(G + 2); + bad += G[3] != 1; + int m[4] = {7, 7, 7, 7}; + long t; + memory(m, m + 2, &t); + bad += m[1] != 1; + int n[4] = {7, 7, 7, 7}; + call(n, n + 2, &t); + bad += n[1] != 1; + return bad; +} From ace696936044a5b5a472bdf7fef3e471403db430 Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 13 Oct 2026 19:52:46 +0200 Subject: [PATCH 10/16] Pass escape bytes to ssePacked. --- src/backend/pass/emit/x86/x86_emit.cpp | 11 +++++++++-- src/backend/pass/emit/x86/x86_encode.cpp | 2 +- src/backend/pass/emit/x86/x86_lower.h | 1 + src/backend/pass/emit/x86/x86_op.h | 6 +++--- src/backend/target/x86/x86_asm.cpp | 18 +++++++++--------- src/backend/target/x86/x86_asm.h | 4 ++-- 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/backend/pass/emit/x86/x86_emit.cpp b/src/backend/pass/emit/x86/x86_emit.cpp index 1b4e1224..dabef450 100644 --- a/src/backend/pass/emit/x86/x86_emit.cpp +++ b/src/backend/pass/emit/x86/x86_emit.cpp @@ -244,9 +244,16 @@ namespace rat { // sse packed vector + // the VArith imm of the instruction pfx 0f esc opc, a pfx or esc of 0 is left out + I64 X86LowerPass::vecDesc(U8 pfx, U8 opc, U8 esc) { + return ((I64)esc << 16) | ((I64)pfx << 8) | opc; + } + void X86LowerPass::varith(VReg d, VReg a, VReg b, U8 pfx, U8 opc, B32 esc38) { - I64 desc = ((I64)(esc38 ? 1 : 0) << 16) | ((I64)pfx << 8) | opc; - farith(X86Op::VArith, d, a, b, 16, desc); + U8 esc = 0; + if(esc38) + esc = 0x38; + farith(X86Op::VArith, d, a, b, 16, vecDesc(pfx, opc, esc)); } void X86LowerPass::vsplat(VReg d, VReg s, U32 esz, B32 isInt) { diff --git a/src/backend/pass/emit/x86/x86_encode.cpp b/src/backend/pass/emit/x86/x86_encode.cpp index 0386adea..45d6f1b9 100644 --- a/src/backend/pass/emit/x86/x86_encode.cpp +++ b/src/backend/pass/emit/x86/x86_encode.cpp @@ -328,7 +328,7 @@ namespace rat { case X86Op::VArith: { U64 imm = (U64)in.imm; U32 d = xmmOf(in.defs[0]), s = xmmOf(in.uses[1]); - return a->ssePacked((U8)(imm >> 8), (U8)imm, d, s, (imm >> 16) != 0); + return a->ssePacked((U8)(imm >> 8), (U8)imm, d, s, (U8)(imm >> 16)); } case X86Op::VSplat: return emitVSplat(in); diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index b02814aa..9ef3befe 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -156,6 +156,7 @@ namespace rat { void cvtf(VReg d, U32 dw, VReg s, U32 sw, U8 pfx, U8 opc, B32 wide); // Cvt, xmm dest void cvti(VReg d, VReg s, U32 sw, U8 pfx, U8 opc, B32 wide); // Cvt, gp dest // sse packed vector + static I64 vecDesc(U8 pfx, U8 opc, U8 esc); void varith(VReg d, VReg a, VReg b, U8 pfx, U8 opc, B32 esc38); void vsplat(VReg d, VReg s, U32 esz, B32 isInt); void vextract(VReg d, VReg s, U32 lane, U32 esz, B32 isInt); diff --git a/src/backend/pass/emit/x86/x86_op.h b/src/backend/pass/emit/x86/x86_op.h index 05462222..a60096ca 100644 --- a/src/backend/pass/emit/x86/x86_op.h +++ b/src/backend/pass/emit/x86/x86_op.h @@ -93,9 +93,9 @@ namespace rat { FCmpFlags, // flags-only ucomis use[0],use[1]; swap in imm2 (cc consumed by a fused Br) Cvt, // SSE convert; pfx/opc/w packed in imm // SSE packed vector (128-bit) - VArith, // dst = dst OP use[1] packed; imm = (0f38 escape << 16) | (prefix << 8) | opcode byte - VSplat, // dst = broadcast use[0] to all lanes; imm = elem bytes, imm2 = int? - VExtract, // dst = lane imm of use[0]; imm2 = (elem bytes << 1) | int? + VArith, // dst = dst OP use[1] packed; imm = (escape << 16) | (prefix << 8) | opcode byte + VSplat, // dst = broadcast use[0] to all lanes; imm = elem bytes, imm2 = int? + VExtract, // dst = lane imm of use[0]; imm2 = (elem bytes << 1) | int? VPackLane, // [vec scratch + imm2>>1 * imm] = use[0] VPack, // dst = the vec scratch slot VPackReg, // dst = movd/movq of int lane use[0] diff --git a/src/backend/target/x86/x86_asm.cpp b/src/backend/target/x86/x86_asm.cpp index 0346246c..b4f2e6b4 100644 --- a/src/backend/target/x86/x86_asm.cpp +++ b/src/backend/target/x86/x86_asm.cpp @@ -422,23 +422,23 @@ namespace rat { b((U8)(0x05 | ((xmm & 7) << 3))); // rip-relative relocs.push_back({reserve32(), sym, RelocKind::Pc32, -4}); } - void Asm::ssePacked(U8 pfx, U8 op, U32 dst, U32 src, B32 esc38) { + void Asm::ssePacked(U8 pfx, U8 op, U32 dst, U32 src, U8 esc) { if(pfx) b(pfx); rex(false, dst, 0, src); b(0x0f); - if(esc38) - b(0x38); + if(esc) + b(esc); b(op); modrmReg(dst, src); } - void Asm::movaps(U32 dst, U32 src) { ssePacked(0, 0x28, dst, src, false); } - void Asm::sseArith(U8 op, U32 w, U32 d, U32 s) { ssePacked(ssePrefixByte(w), op, d, s, false); } - void Asm::ucomis(U32 w, U32 a, U32 bx) { ssePacked(w == 8 ? 0x66 : 0, 0x2e, a, bx, false); } - void Asm::pxor(U32 a, U32 bx) { ssePacked(0x66, 0xef, a, bx, false); } - void Asm::pcmpeqd(U32 a, U32 bx) { ssePacked(0x66, 0x76, a, bx, false); } + void Asm::movaps(U32 dst, U32 src) { ssePacked(0, 0x28, dst, src, 0); } + void Asm::sseArith(U8 op, U32 w, U32 d, U32 s) { ssePacked(ssePrefixByte(w), op, d, s, 0); } + void Asm::ucomis(U32 w, U32 a, U32 bx) { ssePacked(w == 8 ? 0x66 : 0, 0x2e, a, bx, 0); } + void Asm::pxor(U32 a, U32 bx) { ssePacked(0x66, 0xef, a, bx, 0); } + void Asm::pcmpeqd(U32 a, U32 bx) { ssePacked(0x66, 0x76, a, bx, 0); } void Asm::pshufd(U32 dst, U32 src, U8 sel) { - ssePacked(0x66, 0x70, dst, src, false); + ssePacked(0x66, 0x70, dst, src, 0); b(sel); } void Asm::movdXmmGp(U32 xmm, Reg src, B32 wide) { diff --git a/src/backend/target/x86/x86_asm.h b/src/backend/target/x86/x86_asm.h index 761b9bac..b55b54db 100644 --- a/src/backend/target/x86/x86_asm.h +++ b/src/backend/target/x86/x86_asm.h @@ -304,8 +304,8 @@ namespace rat { void storeXmmSib(U32 xmm, Reg base, Reg index, U32 scaleLog2, I32 disp, U32 width); void loadXmmSib(U32 xmm, Reg base, Reg index, U32 scaleLog2, I32 disp, U32 width); void loadXmmRipSym(U32 xmm, const String& sym, U32 width); - // packed SSE op with an explicit mandatory prefix (0 = none), esc38 adds the sse4.1 escape - void ssePacked(U8 pfx, U8 op, U32 dst, U32 src, B32 esc38); + // packed SSE op with an explicit mandatory prefix and escape byte 0x38 or 0x3a, 0 = none + void ssePacked(U8 pfx, U8 op, U32 dst, U32 src, U8 esc); void movaps(U32 dst, U32 src); void sseArith(U8 op, U32 w, U32 d, U32 s); void ucomis(U32 w, U32 a, U32 bx); From ae9309e072fcd60e47f3d88af348a865cd7e3162 Mon Sep 17 00:00:00 2001 From: nella Date: Wed, 14 Oct 2026 10:16:37 +0200 Subject: [PATCH 11/16] Read the copy in x op x. --- src/backend/pass/emit/x86/x86_emit.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/pass/emit/x86/x86_emit.cpp b/src/backend/pass/emit/x86/x86_emit.cpp index dabef450..b39a7947 100644 --- a/src/backend/pass/emit/x86/x86_emit.cpp +++ b/src/backend/pass/emit/x86/x86_emit.cpp @@ -208,7 +208,10 @@ namespace rat { void X86LowerPass::farith(X86Op op, VReg d, VReg a, VReg b, U32 w, I64 desc) { movaps(d, a, w); - put(op, {vr(d, w)}, {vr(d, w), vr(b, w)}, desc); + VReg rhs = b; + if(b == a) + rhs = d; // x op x reads the copy, so a can die at the movaps + put(op, {vr(d, w)}, {vr(d, w), vr(rhs, w)}, desc); } void X86LowerPass::fneg(VReg d, VReg s, U32 w) { From 2c8e188d6e1d7244d67ce4bf15916068da5efb0f Mon Sep 17 00:00:00 2001 From: nella Date: Wed, 14 Oct 2026 11:58:12 +0200 Subject: [PATCH 12/16] Splat loaded ints from xmm. --- src/backend/pass/emit/x86/x86_lower.h | 1 + src/backend/pass/emit/x86/x86_lower_ops.cpp | 22 ++++++-- .../correctness/custom/splat_load_self_op.c | 50 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 src/compiler/test/correctness/custom/splat_load_self_op.c diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index 9ef3befe..5a67c4f8 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -81,6 +81,7 @@ namespace rat { static BinaryNode* asAdd(Node* n); static B32 zextOnlyLoad(const LoadNode* l); static B32 lowHalfOnlyLoad(const LoadNode* l); + static B32 splatOnlyLoad(const Node* n); static U32 opWidth(const Type* t); I32 reserve(U32 bytes, U32 align = 8); diff --git a/src/backend/pass/emit/x86/x86_lower_ops.cpp b/src/backend/pass/emit/x86/x86_lower_ops.cpp index d6509c6d..064054c7 100644 --- a/src/backend/pass/emit/x86/x86_lower_ops.cpp +++ b/src/backend/pass/emit/x86/x86_lower_ops.cpp @@ -39,6 +39,15 @@ namespace rat { return true; } + B32 X86LowerPass::splatOnlyLoad(const Node* n) { + if(!isa(n) || !n->getType()->isInt() || intBits(n->getType()) < 32) + return false; + for(Node* u : n->getUsers()) + if(!isa(u)) + return false; + return n->hasUsers(); + } + void X86LowerPass::emitStore(StoreNode* s) { Node* val = s->getValue(); U32 w = opWidth(val->getType()); @@ -83,6 +92,9 @@ namespace rat { AddrParts a = matchAddr(l->getPointer()); if(isSseTy(l->getType())) { ldf(vregFor(l), w, a); + } else if(splatOnlyLoad(l)) { + vregOf[l->getId()] = fresh(detail::kFp); // movss/movsd, no gp round trip + ldf(vregOf[l->getId()], w, a); } else { B32 sign = l->getType() && l->getType()->isInt() && !zextOnlyLoad(l) && !lowHalfOnlyLoad(l); ld(vregFor(l), w, a, sign); @@ -293,9 +305,13 @@ namespace rat { void X86LowerPass::emitSplat(SplatNode* n) { Type* et = n->getType()->getVecElement(); U32 esz = et->byteSize(ptrBytes); - B32 isInt = !et->isFloat(); - VReg s = isInt ? gpValue(n->getScalar()) : sseValue(n->getScalar()); - vsplat(vregFor(n), s, esz, isInt); + B32 inGp = !et->isFloat() && !splatOnlyLoad(n->getScalar()); + VReg s = kNoVReg; + if(inGp) + s = gpValue(n->getScalar()); + else + s = sseValue(n->getScalar()); + vsplat(vregFor(n), s, esz, inGp); } void X86LowerPass::emitExtract(ExtractNode* n) { diff --git a/src/compiler/test/correctness/custom/splat_load_self_op.c b/src/compiler/test/correctness/custom/splat_load_self_op.c new file mode 100644 index 00000000..715bf8d6 --- /dev/null +++ b/src/compiler/test/correctness/custom/splat_load_self_op.c @@ -0,0 +1,50 @@ +// expect: 0 +// packed loops with an int splat of a loaded scalar (loaded straight into an xmm) and x op x on +// floats (op reads the copy of x) + +int X[64], Y[64]; +float F[64], G[64]; + +__attribute__((noinline)) void saxpy(int* pa) { // *pa may alias Y: one load per iteration + for(int i = 0; i < 64; i += 4) { + int a = *pa; + Y[i] = a * X[i]; + Y[i + 1] = a * X[i + 1]; + Y[i + 2] = a * X[i + 2]; + Y[i + 3] = a * X[i + 3]; + } +} + +__attribute__((noinline)) void self(void) { + for(int i = 0; i < 64; i += 4) { + G[i] = F[i] * F[i]; + G[i + 1] = F[i + 1] * F[i + 1]; + G[i + 2] = F[i + 2] * F[i + 2]; + G[i + 3] = F[i + 3] * F[i + 3]; + } +} + +__attribute__((noinline)) float diff(float x) { return x - x + x / x; } + +int main(void) { + int a = -3; + for(int i = 0; i < 64; i++) { + X[i] = i - 20; + F[i] = (float)i - 0.5f; + } + saxpy(&a); + for(int i = 0; i < 64; i++) + if(Y[i] != -3 * (i - 20)) + return 1; + Y[4] = 2; + saxpy(&Y[4]); // the splat value changes after Y[4] is written + if(Y[3] != 2 * X[3] || Y[4] != 2 * X[4] || Y[7] != 2 * X[7] || Y[8] != 2 * X[4] * X[8]) + return 2; + self(); + for(int i = 0; i < 64; i++) + if(G[i] != F[i] * F[i]) + return 3; + if(diff(2.5f) != 1.0f) + return 4; + return 0; +} From 38b3ca6f9cfe6ffb430d8a1bc8b6357d441fac80 Mon Sep 17 00:00:00 2001 From: nella Date: Wed, 14 Oct 2026 13:44:56 +0200 Subject: [PATCH 13/16] Load partly constant packs from the pool. --- src/backend/pass/emit/x86/x86_emit.cpp | 14 +++--- src/backend/pass/emit/x86/x86_lower.h | 5 +- src/backend/pass/emit/x86/x86_lower_ops.cpp | 56 ++++++++++++++------- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/backend/pass/emit/x86/x86_emit.cpp b/src/backend/pass/emit/x86/x86_emit.cpp index b39a7947..df4175db 100644 --- a/src/backend/pass/emit/x86/x86_emit.cpp +++ b/src/backend/pass/emit/x86/x86_emit.cpp @@ -278,12 +278,14 @@ namespace rat { put(X86Op::VPack, {vr(d, 16)}, {}, (I64)esz, isInt ? 1 : 0); } - void X86LowerPass::vpackReg(VReg d, const List& lanes, U32 esz) { - put(X86Op::VPackReg, {vr(d, 16)}, {lanes[0]}, (I64)esz, 1); - for(U32 i = 1; i < (U32)lanes.size(); ++i) { - Ops uses = {vr(d, 16), lanes[i]}; - put(X86Op::VInsertReg, {vr(d, 16)}, std::move(uses), (I64)esz, (I64)i); - } + // movd/movq of an int lane, the upper lanes zero + void X86LowerPass::vmovd(VReg d, VReg gp, U32 esz) { + put(X86Op::VPackReg, {vr(d, 16)}, {vr(gp)}, (I64)esz, 1); + } + + // pinsrd/pinsrq of an int into lane of d + void X86LowerPass::vinsert(VReg d, VReg gp, U32 esz, U32 lane) { + put(X86Op::VInsertReg, {vr(d, 16)}, {vr(d, 16), vr(gp)}, (I64)esz, (I64)lane); } void X86LowerPass::vshuf(VReg d, VReg s, U8 sel) { diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index 5a67c4f8..9562e6c6 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -162,7 +162,8 @@ namespace rat { void vsplat(VReg d, VReg s, U32 esz, B32 isInt); void vextract(VReg d, VReg s, U32 lane, U32 esz, B32 isInt); void vpackMem(VReg d, const List& lanes, U32 esz, B32 isInt); - void vpackReg(VReg d, const List& lanes, U32 esz); + void vmovd(VReg d, VReg gp, U32 esz); + void vinsert(VReg d, VReg gp, U32 esz, U32 lane); void vshuf(VReg d, VReg s, U8 sel); // x87 void fld(Slot d, VReg addr); @@ -218,8 +219,10 @@ namespace rat { void emitVecBinary(BinaryNode* n); void emitSplat(SplatNode* n); void emitExtract(ExtractNode* n); + List constLanes(PackNode* n, U32 esz); B32 emitConstPack(PackNode* n, U32 esz); void emitPack(PackNode* n); + void emitPackMem(PackNode* n, U32 esz, B32 isInt); void emitSelect(SelectNode* n); const String& vecPoolSym(const List& bytes); VReg gpConst(I64 v); diff --git a/src/backend/pass/emit/x86/x86_lower_ops.cpp b/src/backend/pass/emit/x86/x86_lower_ops.cpp index 064054c7..e510acec 100644 --- a/src/backend/pass/emit/x86/x86_lower_ops.cpp +++ b/src/backend/pass/emit/x86/x86_lower_ops.cpp @@ -324,19 +324,26 @@ namespace rat { vextract(vregFor(n), v, n->getLane(), esz, isInt); } - // all-constant packs come from the constant pool as one movups - B32 X86LowerPass::emitConstPack(PackNode* n, U32 esz) { - U32 w = n->getLaneCount(); - for(U32 i = 0; i < w; ++i) - if(!isa(n->getLane(i))) - return false; + // the constant lanes of n as 16 pool bytes, zero in the other lanes + List X86LowerPass::constLanes(PackNode* n, U32 esz) { List bytes(16, 0); - for(U32 i = 0; i < w; ++i) { - U64 v = (U64)cast(n->getLane(i))->getValue(); + for(U32 i = 0; i < n->getLaneCount(); ++i) { + ConstantNode* c = dyn_cast(n->getLane(i)); + if(!c) + continue; + U64 v = (U64)c->getValue(); for(U32 b = 0; b < esz; ++b) bytes[i * esz + b] = (U8)(v >> (8 * b)); } - ldf(vregFor(n), 16, vecPoolSym(bytes)); + return bytes; + } + + // all-constant packs come from the constant pool as one movups + B32 X86LowerPass::emitConstPack(PackNode* n, U32 esz) { + for(U32 i = 0; i < n->getLaneCount(); ++i) + if(!isa(n->getLane(i))) + return false; + ldf(vregFor(n), 16, vecPoolSym(constLanes(n, esz))); return true; } @@ -347,23 +354,36 @@ namespace rat { B32 isInt = !et->isFloat(); if(emitConstPack(n, esz)) return; - // int lanes with sse4.1 build the vector in-register (VPackReg); float and pre-sse4.1 - // fall back to gathering through the vec scratch slot (VPack) - B32 useReg = isInt && sse41; - if(!useReg && fl->vecScratch == 0) + if(!isInt || !sse41) { + emitPackMem(n, esz, isInt); + return; + } + List bytes = constLanes(n, esz); + B32 nonZero = std::count(bytes.begin(), bytes.end(), 0) < 16; + B32 fromPool = nonZero || isa(n->getLane(0)); + VReg d = vregFor(n); + if(fromPool) + ldf(d, 16, vecPoolSym(bytes)); + else + vmovd(d, gpValue(n->getLane(0)), esz); + for(U32 i = 0; i < w; ++i) + if(!isa(n->getLane(i)) && (i || fromPool)) + vinsert(d, gpValue(n->getLane(i)), esz, i); + } + + // float lanes, and int lanes without sse4.1, go through the vec scratch slot + void X86LowerPass::emitPackMem(PackNode* n, U32 esz, B32 isInt) { + if(fl->vecScratch == 0) fl->vecScratch = reserve(16); List lanes; - for(U32 i = 0; i < w; ++i) { + for(U32 i = 0; i < n->getLaneCount(); ++i) { Node* lane = n->getLane(i); if(isInt) lanes.push_back(MachineOperand::vr(gpValue(lane))); else lanes.push_back(MachineOperand::vr(sseValue(lane), esz)); } - if(useReg) - vpackReg(vregFor(n), lanes, esz); - else - vpackMem(vregFor(n), lanes, esz, isInt); + vpackMem(vregFor(n), lanes, esz, isInt); } // a fresh register holding a constant too wide for an ALU immediate From 4fe678e104be236acd5713b515597b01b0c6eb30 Mon Sep 17 00:00:00 2001 From: nella Date: Wed, 14 Oct 2026 15:02:29 +0200 Subject: [PATCH 14/16] Build float packs in registers. --- src/backend/pass/emit/x86/x86_emit.cpp | 14 +++++- src/backend/pass/emit/x86/x86_encode.cpp | 8 ++++ src/backend/pass/emit/x86/x86_lower.cpp | 2 +- src/backend/pass/emit/x86/x86_lower.h | 5 +++ src/backend/pass/emit/x86/x86_lower_ops.cpp | 49 +++++++++++++++++++-- src/backend/pass/emit/x86/x86_op.cpp | 1 + src/backend/pass/emit/x86/x86_op.h | 8 +++- 7 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/backend/pass/emit/x86/x86_emit.cpp b/src/backend/pass/emit/x86/x86_emit.cpp index df4175db..d617b91f 100644 --- a/src/backend/pass/emit/x86/x86_emit.cpp +++ b/src/backend/pass/emit/x86/x86_emit.cpp @@ -247,7 +247,7 @@ namespace rat { // sse packed vector - // the VArith imm of the instruction pfx 0f esc opc, a pfx or esc of 0 is left out + // the VArith/VUnary imm of the instruction pfx 0f esc opc, a pfx or esc of 0 is left out I64 X86LowerPass::vecDesc(U8 pfx, U8 opc, U8 esc) { return ((I64)esc << 16) | ((I64)pfx << 8) | opc; } @@ -259,6 +259,18 @@ namespace rat { farith(X86Op::VArith, d, a, b, 16, vecDesc(pfx, opc, esc)); } + // d = d op s in place (insertps, movsd, movlhps), with an imm8 unless imm8 < 0 + void X86LowerPass::vmerge(VReg d, VReg s, U8 pfx, U8 opc, U8 esc, I32 imm8) { + I64 desc = vecDesc(pfx, opc, esc); + if(imm8 >= 0) + desc |= detail::kVecImm8; + put(X86Op::VArith, {vr(d, 16)}, {vr(d, 16), vr(s, 16)}, desc, imm8); + } + + void X86LowerPass::vunary(VReg d, VReg s, U8 pfx, U8 opc, U8 esc) { + put(X86Op::VUnary, {vr(d, 16)}, {vr(s, 16)}, vecDesc(pfx, opc, esc)); + } + void X86LowerPass::vsplat(VReg d, VReg s, U32 esz, B32 isInt) { Ops uses = {vr(s, isInt ? 8 : esz)}; put(X86Op::VSplat, {vr(d, 16)}, std::move(uses), (I64)esz, isInt ? 1 : 0); diff --git a/src/backend/pass/emit/x86/x86_encode.cpp b/src/backend/pass/emit/x86/x86_encode.cpp index 45d6f1b9..4c9373cf 100644 --- a/src/backend/pass/emit/x86/x86_encode.cpp +++ b/src/backend/pass/emit/x86/x86_encode.cpp @@ -328,6 +328,14 @@ namespace rat { case X86Op::VArith: { U64 imm = (U64)in.imm; U32 d = xmmOf(in.defs[0]), s = xmmOf(in.uses[1]); + a->ssePacked((U8)(imm >> 8), (U8)imm, d, s, (U8)(imm >> 16)); + if(imm & detail::kVecImm8) + a->b((U8)in.imm2); + return; + } + case X86Op::VUnary: { + U64 imm = (U64)in.imm; + U32 d = xmmOf(in.defs[0]), s = xmmOf(in.uses[0]); return a->ssePacked((U8)(imm >> 8), (U8)imm, d, s, (U8)(imm >> 16)); } case X86Op::VSplat: diff --git a/src/backend/pass/emit/x86/x86_lower.cpp b/src/backend/pass/emit/x86/x86_lower.cpp index dee3408f..059611f8 100644 --- a/src/backend/pass/emit/x86/x86_lower.cpp +++ b/src/backend/pass/emit/x86/x86_lower.cpp @@ -362,7 +362,7 @@ namespace rat { lea(vregFor(n), cast(n)->getSymbol()); return; case Opcode::Constant: - if(isSseTy(n->getType())) + if(isSseTy(n->getType()) && !packOnlyConst(n)) fpConstLoad(cast(n), vregFor(n)); return; case Opcode::Store: diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index 9562e6c6..e6f3d5e5 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -82,6 +82,7 @@ namespace rat { static B32 zextOnlyLoad(const LoadNode* l); static B32 lowHalfOnlyLoad(const LoadNode* l); static B32 splatOnlyLoad(const Node* n); + static B32 packOnlyConst(const Node* n); static U32 opWidth(const Type* t); I32 reserve(U32 bytes, U32 align = 8); @@ -159,6 +160,8 @@ namespace rat { // sse packed vector static I64 vecDesc(U8 pfx, U8 opc, U8 esc); void varith(VReg d, VReg a, VReg b, U8 pfx, U8 opc, B32 esc38); + void vmerge(VReg d, VReg s, U8 pfx, U8 opc, U8 esc, I32 imm8); + void vunary(VReg d, VReg s, U8 pfx, U8 opc, U8 esc); void vsplat(VReg d, VReg s, U32 esz, B32 isInt); void vextract(VReg d, VReg s, U32 lane, U32 esz, B32 isInt); void vpackMem(VReg d, const List& lanes, U32 esz, B32 isInt); @@ -223,6 +226,8 @@ namespace rat { B32 emitConstPack(PackNode* n, U32 esz); void emitPack(PackNode* n); void emitPackMem(PackNode* n, U32 esz, B32 isInt); + void packLaneZero(PackNode* n, VReg d, U32 esz, B32 isInt); + void insertLane(VReg d, Node* lane, U32 i, U32 esz, B32 isInt); void emitSelect(SelectNode* n); const String& vecPoolSym(const List& bytes); VReg gpConst(I64 v); diff --git a/src/backend/pass/emit/x86/x86_lower_ops.cpp b/src/backend/pass/emit/x86/x86_lower_ops.cpp index e510acec..b15aeac9 100644 --- a/src/backend/pass/emit/x86/x86_lower_ops.cpp +++ b/src/backend/pass/emit/x86/x86_lower_ops.cpp @@ -48,6 +48,13 @@ namespace rat { return n->hasUsers(); } + B32 X86LowerPass::packOnlyConst(const Node* n) { + for(Node* u : n->getUsers()) + if(!isa(u)) + return false; + return n->hasUsers(); + } + void X86LowerPass::emitStore(StoreNode* s) { Node* val = s->getValue(); U32 w = opWidth(val->getType()); @@ -354,7 +361,7 @@ namespace rat { B32 isInt = !et->isFloat(); if(emitConstPack(n, esz)) return; - if(!isInt || !sse41) { + if(!sse41) { emitPackMem(n, esz, isInt); return; } @@ -365,13 +372,13 @@ namespace rat { if(fromPool) ldf(d, 16, vecPoolSym(bytes)); else - vmovd(d, gpValue(n->getLane(0)), esz); + packLaneZero(n, d, esz, isInt); for(U32 i = 0; i < w; ++i) if(!isa(n->getLane(i)) && (i || fromPool)) - vinsert(d, gpValue(n->getLane(i)), esz, i); + insertLane(d, n->getLane(i), i, esz, isInt); } - // float lanes, and int lanes without sse4.1, go through the vec scratch slot + // without sse4.1 the lanes go through the vec scratch slot void X86LowerPass::emitPackMem(PackNode* n, U32 esz, B32 isInt) { if(fl->vecScratch == 0) fl->vecScratch = reserve(16); @@ -386,6 +393,40 @@ namespace rat { vpackMem(vregFor(n), lanes, esz, isInt); } + void X86LowerPass::packLaneZero(PackNode* n, VReg d, U32 esz, B32 isInt) { + Node* lane = n->getLane(0); + if(isInt) { + vmovd(d, gpValue(lane), esz); + return; + } + U8 zero = 0; // insertps zero mask + for(U32 i = 1; i < n->getLaneCount(); ++i) + if(isa(n->getLane(i))) + zero |= (U8)(1 << i); + VReg s = sseValue(lane); + if(zero && esz == 8) { + vunary(d, s, 0xf3, 0x7e, 0); // movq + return; + } + movaps(d, s, esz); + if(zero) + vmerge(d, d, 0x66, 0x21, 0x3a, zero); // insertps, zeroing the constant lanes + } + + void X86LowerPass::insertLane(VReg d, Node* lane, U32 i, U32 esz, B32 isInt) { + if(isInt) { + vinsert(d, gpValue(lane), esz, i); + return; + } + VReg s = sseValue(lane); + if(esz == 4) + vmerge(d, s, 0x66, 0x21, 0x3a, (I32)(i << 4)); + else if(i == 0) + vmerge(d, s, 0xf2, 0x10, 0, -1); + else + vmerge(d, s, 0, 0x16, 0, -1); + } + // a fresh register holding a constant too wide for an ALU immediate VReg X86LowerPass::gpConst(I64 v) { VReg t = fresh(detail::kGp); diff --git a/src/backend/pass/emit/x86/x86_op.cpp b/src/backend/pass/emit/x86/x86_op.cpp index d11b6044..f9098254 100644 --- a/src/backend/pass/emit/x86/x86_op.cpp +++ b/src/backend/pass/emit/x86/x86_op.cpp @@ -63,6 +63,7 @@ namespace rat { {"cvt", kFp}, // sse packed vector {"varith", kFp}, + {"vunary", kFp}, {"vsplat", kFp}, {"vextract", kFp}, {"vpacklane", kFp}, diff --git a/src/backend/pass/emit/x86/x86_op.h b/src/backend/pass/emit/x86/x86_op.h index a60096ca..e8e40e74 100644 --- a/src/backend/pass/emit/x86/x86_op.h +++ b/src/backend/pass/emit/x86/x86_op.h @@ -33,6 +33,9 @@ namespace rat { constexpr U8 kFpSwap[] = {0, 0, 1, 1, 0, 0}; constexpr U8 kSseOp[] = {0x58, 0x5c, 0x59, 0x5e}; + + // VArith desc bit, set when imm2 holds an imm8 that follows the modrm + constexpr I64 kVecImm8 = (I64)1 << 24; } // namespace detail enum class X86Op : U32 { @@ -93,7 +96,10 @@ namespace rat { FCmpFlags, // flags-only ucomis use[0],use[1]; swap in imm2 (cc consumed by a fused Br) Cvt, // SSE convert; pfx/opc/w packed in imm // SSE packed vector (128-bit) - VArith, // dst = dst OP use[1] packed; imm = (escape << 16) | (prefix << 8) | opcode byte + // dst = dst OP use[1] packed, imm = (escape << 16) | (prefix << 8) | opcode, plus kVecImm8 when + // imm2 holds an imm8 + VArith, + VUnary, // dst = OP use[0] packed, imm as VArith VSplat, // dst = broadcast use[0] to all lanes; imm = elem bytes, imm2 = int? VExtract, // dst = lane imm of use[0]; imm2 = (elem bytes << 1) | int? VPackLane, // [vec scratch + imm2>>1 * imm] = use[0] From e6301982c7455af43ce964c47805702899ab7b92 Mon Sep 17 00:00:00 2001 From: nella Date: Wed, 14 Oct 2026 17:36:44 +0200 Subject: [PATCH 15/16] Give shuffles two sources. --- src/backend/ir/node.cpp | 12 +++--- src/backend/ir/node.h | 16 ++++--- src/backend/ir/opcode.cpp | 2 +- src/backend/ir/text_parser.cpp | 7 ++- src/backend/pass/emit/graph_emitter.cpp | 1 - src/backend/pass/emit/x86/x86_emit.cpp | 10 ++++- src/backend/pass/emit/x86/x86_lower.cpp | 6 +-- src/backend/pass/emit/x86/x86_lower.h | 6 ++- src/backend/pass/emit/x86/x86_lower_ops.cpp | 48 +++++++++++++++++++++ src/backend/pass/opt/slp/slp_packer.cpp | 14 +++--- src/backend/pass/opt/slp/slp_reduce.cpp | 7 ++- src/backend/pass/verify.cpp | 12 ++++-- src/backend/test/slp_reduce_dot_i32.rat | 4 +- 13 files changed, 109 insertions(+), 36 deletions(-) diff --git a/src/backend/ir/node.cpp b/src/backend/ir/node.cpp index 8c809926..de6faae8 100644 --- a/src/backend/ir/node.cpp +++ b/src/backend/ir/node.cpp @@ -358,12 +358,14 @@ namespace rat { Node* SelectNode::getTrue() const { return getInput(1); } Node* SelectNode::getFalse() const { return getInput(2); } - ShuffleNode::ShuffleNode(Function& fn, Type* vecType, Node* vector, U8 sel) - : Node(fn, Opcode::Shuffle, vecType, {vector}), + ShuffleNode::ShuffleNode(Function& fn, Type* vecType, Node* first, Node* second, U16 sel) + : Node(fn, Opcode::Shuffle, vecType, {first, second}), sel(sel) {} - Node* ShuffleNode::getVector() const { return getInput(0); } - U8 ShuffleNode::getSelector() const { return sel; } + Node* ShuffleNode::getFirst() const { return getInput(0); } + Node* ShuffleNode::getSecond() const { return getInput(1); } + U16 ShuffleNode::getSelector() const { return sel; } + U32 ShuffleNode::getSource(U32 lane) const { return sel >> (kSourceBits * lane) & kSourceMask; } B32 isControlNode(const Node* n) { switch(n->getOpcode()) { @@ -450,7 +452,7 @@ namespace rat { case Opcode::Pack: return into.create(t, nulls); case Opcode::Shuffle: - return into.create(t, nullptr, cast(n)->getSelector()); + return into.create(t, nullptr, nullptr, cast(n)->getSelector()); case Opcode::Select: return into.create(t, nullptr, nullptr, nullptr); default: diff --git a/src/backend/ir/node.h b/src/backend/ir/node.h index 326a6243..ab9ee9ec 100644 --- a/src/backend/ir/node.h +++ b/src/backend/ir/node.h @@ -334,14 +334,20 @@ namespace rat { Node* getFalse() const; }; - // permute 32-bit lanes by a pshufd-style selector + // lane i of the result is lane (sel >> 4i) & 15 of the lanes of first followed by those of second struct ShuffleNode : Node { - ShuffleNode(Function& fn, Type* vecType, Node* vector, U8 sel); + static constexpr U32 kSourceBits = 4; + static constexpr U32 kSourceMask = (1u << kSourceBits) - 1; + static constexpr U32 kMaxLanes = 4; // a 16-bit selector holds four sources - Node* getVector() const; - U8 getSelector() const; + ShuffleNode(Function& fn, Type* vecType, Node* first, Node* second, U16 sel); + + Node* getFirst() const; + Node* getSecond() const; + U16 getSelector() const; + U32 getSource(U32 lane) const; private: - U8 sel; + U16 sel; }; // address of a module-level symbol (global or function), as a pointer diff --git a/src/backend/ir/opcode.cpp b/src/backend/ir/opcode.cpp index a9b5bde4..1b330b5b 100644 --- a/src/backend/ir/opcode.cpp +++ b/src/backend/ir/opcode.cpp @@ -72,7 +72,7 @@ namespace rat { {"splat", 0, 0, 0, -1, 1, 1, OpClass::None}, {"extract", 0, 0, 0, -1, 1, 1, OpClass::None}, {"pack", 0, 0, 0, -1, 1, -1, OpClass::None}, - {"shuffle", 0, 0, 0, -1, 1, 1, OpClass::None}, + {"shuffle", 0, 0, 0, -1, 2, 2, OpClass::None}, {"select", 0, 0, 0, -1, 3, 3, OpClass::None}, }; static_assert(sizeof(kOpcodeInfo) / sizeof(kOpcodeInfo[0]) == (U32)Opcode::Select + 1, diff --git a/src/backend/ir/text_parser.cpp b/src/backend/ir/text_parser.cpp index 920fd491..a078ec7d 100644 --- a/src/backend/ir/text_parser.cpp +++ b/src/backend/ir/text_parser.cpp @@ -471,6 +471,11 @@ namespace rat { // reuse the proj payload slot for the selector if(!parseIndex(laneTok.substr(1), pn.projIndex)) return fail("lane selector out of range: " + line); + if(pn.op == Opcode::Shuffle) { + pn.operands = parseVRefs(remainder); + B32 ok = pn.operands.size() == 2 && pn.projIndex <= UINT16_MAX; + return ok || fail("shuffle takes a 16-bit selector and two vectors: " + line); + } return singleRef(remainder, "lane op must reference exactly one vector: " + line, pn); } @@ -596,7 +601,7 @@ namespace rat { case Opcode::Extract: return fn->create(pn.ty, in[0], pn.projIndex); case Opcode::Shuffle: - return fn->create(pn.ty, in[0], (U8)pn.projIndex); + return fn->create(pn.ty, in[0], in[1], (U16)pn.projIndex); case Opcode::Select: return fn->create(pn.ty, in[0], in[1], in[2]); case Opcode::Pack: diff --git a/src/backend/pass/emit/graph_emitter.cpp b/src/backend/pass/emit/graph_emitter.cpp index 4ac3e6f5..3280dcf1 100644 --- a/src/backend/pass/emit/graph_emitter.cpp +++ b/src/backend/pass/emit/graph_emitter.cpp @@ -70,7 +70,6 @@ namespace rat { case Opcode::Splat: return "val"; case Opcode::Extract: - case Opcode::Shuffle: return "vec"; default: return std::to_string(i); diff --git a/src/backend/pass/emit/x86/x86_emit.cpp b/src/backend/pass/emit/x86/x86_emit.cpp index d617b91f..739e1bb7 100644 --- a/src/backend/pass/emit/x86/x86_emit.cpp +++ b/src/backend/pass/emit/x86/x86_emit.cpp @@ -206,12 +206,12 @@ namespace rat { put(X86Op::FStore, {}, std::move(uses), a.disp, sibBits(0, a)); } - void X86LowerPass::farith(X86Op op, VReg d, VReg a, VReg b, U32 w, I64 desc) { + MachineInstr& X86LowerPass::farith(X86Op op, VReg d, VReg a, VReg b, U32 w, I64 desc) { movaps(d, a, w); VReg rhs = b; if(b == a) rhs = d; // x op x reads the copy, so a can die at the movaps - put(op, {vr(d, w)}, {vr(d, w), vr(rhs, w)}, desc); + return put(op, {vr(d, w)}, {vr(d, w), vr(rhs, w)}, desc); } void X86LowerPass::fneg(VReg d, VReg s, U32 w) { @@ -259,6 +259,12 @@ namespace rat { farith(X86Op::VArith, d, a, b, 16, vecDesc(pfx, opc, esc)); } + // d = a op b with an immediate operand, esc is the 0f38/0f3a escape byte or 0 + void X86LowerPass::varithImm(VReg d, VReg a, VReg b, U8 pfx, U8 opc, U8 esc, U8 imm8) { + I64 desc = vecDesc(pfx, opc, esc) | detail::kVecImm8; + farith(X86Op::VArith, d, a, b, 16, desc).imm2 = imm8; + } + // d = d op s in place (insertps, movsd, movlhps), with an imm8 unless imm8 < 0 void X86LowerPass::vmerge(VReg d, VReg s, U8 pfx, U8 opc, U8 esc, I32 imm8) { I64 desc = vecDesc(pfx, opc, esc); diff --git a/src/backend/pass/emit/x86/x86_lower.cpp b/src/backend/pass/emit/x86/x86_lower.cpp index 059611f8..666208ab 100644 --- a/src/backend/pass/emit/x86/x86_lower.cpp +++ b/src/backend/pass/emit/x86/x86_lower.cpp @@ -399,11 +399,9 @@ namespace rat { case Opcode::Pack: emitPack(cast(n)); return; - case Opcode::Shuffle: { - VReg v = sseValue(cast(n)->getVector()); - vshuf(vregFor(n), v, cast(n)->getSelector()); + case Opcode::Shuffle: + emitShuffle(cast(n)); return; - } case Opcode::Select: emitSelect(cast(n)); return; diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index e6f3d5e5..3ecd8cbf 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -22,6 +22,7 @@ namespace rat { struct PackNode; struct ProjNode; struct SelectNode; + struct ShuffleNode; struct ReturnNode; struct SplatNode; struct StoreNode; @@ -149,7 +150,7 @@ namespace rat { void ldf(VReg d, U32 w, const String& s); void ldf(VReg d, U32 w, const AddrParts& a); void stf(const AddrParts& a, VReg s, U32 w); - void farith(X86Op op, VReg d, VReg a, VReg b, U32 w, I64 desc); + MachineInstr& farith(X86Op op, VReg d, VReg a, VReg b, U32 w, I64 desc); void fneg(VReg d, VReg s, U32 w); void fsqrt(VReg d, VReg s, U32 w); void fabs_(VReg d, VReg s, U32 w); @@ -160,6 +161,7 @@ namespace rat { // sse packed vector static I64 vecDesc(U8 pfx, U8 opc, U8 esc); void varith(VReg d, VReg a, VReg b, U8 pfx, U8 opc, B32 esc38); + void varithImm(VReg d, VReg a, VReg b, U8 pfx, U8 opc, U8 esc, U8 imm8); void vmerge(VReg d, VReg s, U8 pfx, U8 opc, U8 esc, I32 imm8); void vunary(VReg d, VReg s, U8 pfx, U8 opc, U8 esc); void vsplat(VReg d, VReg s, U32 esz, B32 isInt); @@ -228,6 +230,8 @@ namespace rat { void emitPackMem(PackNode* n, U32 esz, B32 isInt); void packLaneZero(PackNode* n, VReg d, U32 esz, B32 isInt); void insertLane(VReg d, Node* lane, U32 i, U32 esz, B32 isInt); + static void shuffleDwords(const ShuffleNode* n, U8& sel, U8& mask, B32& inPlace); + void emitShuffle(ShuffleNode* n); void emitSelect(SelectNode* n); const String& vecPoolSym(const List& bytes); VReg gpConst(I64 v); diff --git a/src/backend/pass/emit/x86/x86_lower_ops.cpp b/src/backend/pass/emit/x86/x86_lower_ops.cpp index b15aeac9..7d2a6790 100644 --- a/src/backend/pass/emit/x86/x86_lower_ops.cpp +++ b/src/backend/pass/emit/x86/x86_lower_ops.cpp @@ -427,6 +427,54 @@ namespace rat { vmerge(d, s, 0, 0x16, 0, -1); } + void X86LowerPass::shuffleDwords(const ShuffleNode* n, U8& sel, U8& mask, B32& inPlace) { + Type* vt = n->getType(); + U32 lanes = vt->getVecLanes(); + U32 per = opWidth(vt->getVecElement()) / 4; + U32 src[4] = {0, 1, 2, 3}; + for(U32 i = 0; i < lanes; ++i) { + U32 s = n->getSource(i); + for(U32 k = 0; k < per; ++k) + src[i * per + k] = s / lanes * 4 + s % lanes * per + k; + } + sel = 0; + mask = 0; + inPlace = true; + for(U32 i = 0; i < 4; ++i) { + if(n->getFirst() == n->getSecond()) + src[i] &= 3; + sel |= (U8)((src[i] & 3) << (2 * i)); + mask |= (U8)((src[i] >> 2) << i); + inPlace &= (src[i] & 3) == i; + } + } + + void X86LowerPass::emitShuffle(ShuffleNode* n) { + U8 sel = 0, mask = 0; + B32 inPlace = true; + shuffleDwords(n, sel, mask, inPlace); + VReg a = sseValue(n->getFirst()); + VReg b = sseValue(n->getSecond()); + VReg d = vregFor(n); + if(inPlace && mask != 0 && mask != 15) { + varithImm(d, a, b, 0x66, 0x0c, 0x3a, mask); // blendps + return; + } + if(mask == 15 || mask == 3) + std::swap(a, b); // the low half comes from second + if(mask == 0 || mask == 15) { + vshuf(d, a, sel); + } else if(mask == 12 || mask == 3) { + varithImm(d, a, b, 0, 0xc6, 0, sel); // shufps + } else { + VReg ta = fresh(detail::kFp); + VReg tb = fresh(detail::kFp); + vshuf(ta, a, sel); + vshuf(tb, b, sel); + varithImm(d, ta, tb, 0x66, 0x0c, 0x3a, mask); + } + } + // a fresh register holding a constant too wide for an ALU immediate VReg X86LowerPass::gpConst(I64 v) { VReg t = fresh(detail::kGp); diff --git a/src/backend/pass/opt/slp/slp_packer.cpp b/src/backend/pass/opt/slp/slp_packer.cpp index b1d497fc..3d0cea30 100644 --- a/src/backend/pass/opt/slp/slp_packer.cpp +++ b/src/backend/pass/opt/slp/slp_packer.cpp @@ -364,15 +364,11 @@ namespace rat { Node* wide = fn.create( vecTy, cs[i].load->getControl(), cs[i].load->getMemory(), cs[i].load->getPointer()); for(U32 j = 0; j < w; ++j) { - // broadcast lane j: pshufd selector for 4-byte lanes, unpack hi/lo for 8-byte - U8 sel; - if(esz == 4) - sel = (U8)(j * 0x55); - else if(j) - sel = 0xee; - else - sel = 0x44; - cs[i + j].splat->replaceAllUsesWith(fn.create(vecTy, wide, sel)); + // broadcast lane j + U16 sel = 0; + for(U32 k = 0; k < w; ++k) + sel |= (U16)(j << (ShuffleNode::kSourceBits * k)); + cs[i + j].splat->replaceAllUsesWith(fn.create(vecTy, wide, wide, sel)); } i += w; } diff --git a/src/backend/pass/opt/slp/slp_reduce.cpp b/src/backend/pass/opt/slp/slp_reduce.cpp index 76bcbd59..b2078d16 100644 --- a/src/backend/pass/opt/slp/slp_reduce.cpp +++ b/src/backend/pass/opt/slp/slp_reduce.cpp @@ -49,10 +49,13 @@ namespace rat { // log2 shuffle+add finish Node* slp::Slp::emitHsum(Node* acc, Opcode addOp, Type* vecTy, U32 w) { - Node* s1 = fn.create(vecTy, acc, (U8)0x4e); // swap 64-bit halves + U16 half = 0x01; // swap the two lanes + if(w == 4) + half = 0x1032; // swap the 64-bit halves + Node* s1 = fn.create(vecTy, acc, acc, half); acc = fn.create(addOp, vecTy, acc, s1); if(w == 4) { - Node* s2 = fn.create(vecTy, acc, (U8)0xb1); // swap 32-bit pairs + Node* s2 = fn.create(vecTy, acc, acc, (U16)0x2301); // swap 32-bit pairs acc = fn.create(addOp, vecTy, acc, s2); } return acc; diff --git a/src/backend/pass/verify.cpp b/src/backend/pass/verify.cpp index 5841a46a..3a5fb359 100644 --- a/src/backend/pass/verify.cpp +++ b/src/backend/pass/verify.cpp @@ -351,10 +351,16 @@ namespace rat { err(n, "Pack lane " + std::to_string(i) + " type does not match the vector element"); break; default: - if(!t->isVec()) - err(n, "Shuffle type must be a vector"); - else if(in != t) + if(!t->isVec() || t->getVecLanes() > ShuffleNode::kMaxLanes) { + String maxLanes = std::to_string(ShuffleNode::kMaxLanes); + err(n, "Shuffle type must be a vector of at most " + maxLanes + " lanes"); + break; + } + if(in != t || n->getInput(1)->getType() != t) err(n, "Shuffle operand type does not match the result vector"); + for(U32 i = 0; i < t->getVecLanes(); ++i) + if(cast(n)->getSource(i) >= 2 * t->getVecLanes()) + err(n, "Shuffle lane source is out of range"); break; } } diff --git a/src/backend/test/slp_reduce_dot_i32.rat b/src/backend/test/slp_reduce_dot_i32.rat index bc0cafd4..dd7b74e6 100644 --- a/src/backend/test/slp_reduce_dot_i32.rat +++ b/src/backend/test/slp_reduce_dot_i32.rat @@ -83,9 +83,9 @@ func dot(ptr, ptr) -> i32 { v14 = load : <4 x i32> v2, v3, v13 v15 = mul : <4 x i32> v11, v14 v16 = add : <4 x i32> v8, v15 - v17 = shuffle : <4 x i32> #78 v16 + v17 = shuffle : <4 x i32> #4146 v16, v16 v18 = add : <4 x i32> v16, v17 - v19 = shuffle : <4 x i32> #177 v18 + v19 = shuffle : <4 x i32> #8961 v18, v18 v20 = add : <4 x i32> v18, v19 v21 = extract : i32 #0 v20 v22 = return : ctrl v2, v3, v21 From 04ff93e369f9a34ea5f1ac6cff0f0f049a3ac317 Mon Sep 17 00:00:00 2001 From: nella Date: Wed, 14 Oct 2026 20:15:08 +0200 Subject: [PATCH 16/16] Lower addsub shuffles. --- src/backend/pass/emit/x86/x86_lower.cpp | 2 + src/backend/pass/emit/x86/x86_lower.h | 3 ++ src/backend/pass/emit/x86/x86_lower_ops.cpp | 41 +++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/src/backend/pass/emit/x86/x86_lower.cpp b/src/backend/pass/emit/x86/x86_lower.cpp index 666208ab..668d628a 100644 --- a/src/backend/pass/emit/x86/x86_lower.cpp +++ b/src/backend/pass/emit/x86/x86_lower.cpp @@ -423,6 +423,8 @@ namespace rat { return; // folded into base+index*scale+disp of every using load/store if(addressOnlyScale(n)) return; // folded into the SIB scale of every using address + if(addSubOnly(n)) + return; // each addsub shuffle computes it emitBinary(cast(n)); } } diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index 3ecd8cbf..eb01f15c 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -84,6 +84,8 @@ namespace rat { static B32 lowHalfOnlyLoad(const LoadNode* l); static B32 splatOnlyLoad(const Node* n); static B32 packOnlyConst(const Node* n); + static BinaryNode* addSubOf(const ShuffleNode* n); + static B32 addSubOnly(const Node* n); static U32 opWidth(const Type* t); I32 reserve(U32 bytes, U32 align = 8); @@ -230,6 +232,7 @@ namespace rat { void emitPackMem(PackNode* n, U32 esz, B32 isInt); void packLaneZero(PackNode* n, VReg d, U32 esz, B32 isInt); void insertLane(VReg d, Node* lane, U32 i, U32 esz, B32 isInt); + void emitAddSub(ShuffleNode* n, BinaryNode* sub); static void shuffleDwords(const ShuffleNode* n, U8& sel, U8& mask, B32& inPlace); void emitShuffle(ShuffleNode* n); void emitSelect(SelectNode* n); diff --git a/src/backend/pass/emit/x86/x86_lower_ops.cpp b/src/backend/pass/emit/x86/x86_lower_ops.cpp index 7d2a6790..0d0c140e 100644 --- a/src/backend/pass/emit/x86/x86_lower_ops.cpp +++ b/src/backend/pass/emit/x86/x86_lower_ops.cpp @@ -55,6 +55,36 @@ namespace rat { return n->hasUsers(); } + BinaryNode* X86LowerPass::addSubOf(const ShuffleNode* n) { + U32 lanes = n->getType()->getVecLanes(); + Node* from[2] = {n->getFirst(), n->getSecond()}; + Node* parity[2] = {nullptr, nullptr}; // source of the even and the odd lanes + for(U32 i = 0; i < lanes; ++i) { + U32 s = n->getSource(i); + Node*& want = parity[i % 2]; + if(s % lanes != i || (want && want != from[s / lanes])) + return nullptr; + want = from[s / lanes]; + } + BinaryNode* sub = dyn_cast(parity[0]); + BinaryNode* add = dyn_cast(parity[1]); + if(!sub || !add || sub->getOpcode() != Opcode::FSub || add->getOpcode() != Opcode::FAdd) + return nullptr; + if(sub->getLHS() != add->getLHS() || sub->getRHS() != add->getRHS()) + return nullptr; + return sub; + } + + B32 X86LowerPass::addSubOnly(const Node* n) { + Opcode op = n->getOpcode(); + if(!n->getType()->isVec() || (op != Opcode::FAdd && op != Opcode::FSub)) + return false; + for(Node* u : n->getUsers()) + if(!isa(u) || !addSubOf(cast(u))) + return false; + return n->hasUsers(); + } + void X86LowerPass::emitStore(StoreNode* s) { Node* val = s->getValue(); U32 w = opWidth(val->getType()); @@ -427,6 +457,13 @@ namespace rat { vmerge(d, s, 0, 0x16, 0, -1); } + void X86LowerPass::emitAddSub(ShuffleNode* n, BinaryNode* sub) { + U8 pfx = 0xf2; // addsubps + if(opWidth(n->getType()->getVecElement()) == 8) + pfx = 0x66; // addsubpd + varith(vregFor(n), sseValue(sub->getLHS()), sseValue(sub->getRHS()), pfx, 0xd0, false); + } + void X86LowerPass::shuffleDwords(const ShuffleNode* n, U8& sel, U8& mask, B32& inPlace) { Type* vt = n->getType(); U32 lanes = vt->getVecLanes(); @@ -450,6 +487,10 @@ namespace rat { } void X86LowerPass::emitShuffle(ShuffleNode* n) { + if(BinaryNode* sub = addSubOf(n)) { + emitAddSub(n, sub); + return; + } U8 sel = 0, mask = 0; B32 inPlace = true; shuffleDwords(n, sel, mask, inPlace);