diff --git a/jlm/llvm/ir/Trace.cpp b/jlm/llvm/ir/Trace.cpp index 1cff960c1..65f4e3e00 100644 --- a/jlm/llvm/ir/Trace.cpp +++ b/jlm/llvm/ir/Trace.cpp @@ -28,33 +28,39 @@ OutputTracer::OutputTracer() : rvsdg::OutputTracer() {} -rvsdg::Output & -OutputTracer::traceStep(rvsdg::Output & output, const rvsdg::Region * withinRegion) +OutputTracer::TraceStepResult +OutputTracer::traceStep( + rvsdg::Output & output, + BackEdgeState backEdgeState, + const rvsdg::Region * withinRegion) { - auto & trace1 = rvsdg::OutputTracer::traceStep(output, withinRegion); + const auto trace1 = rvsdg::OutputTracer::traceStep(output, backEdgeState, withinRegion); + auto & trace1Output = trace1.getOutput(); if (const auto [node, ioBarrierOp] = - rvsdg::TryGetSimpleNodeAndOptionalOp(trace1); + rvsdg::TryGetSimpleNodeAndOptionalOp(trace1Output); node && ioBarrierOp) { - return *IOBarrierOperation::BarredInput(*node).origin(); + return TraceStepResult::createStepResult(*IOBarrierOperation::BarredInput(*node).origin()); } // If enabled, try tracing through the memory states of load nodes if (traceThroughLoadedStates_) { - if (const auto [node, loadOp] = rvsdg::TryGetSimpleNodeAndOptionalOp(trace1); + if (const auto [node, loadOp] = + rvsdg::TryGetSimpleNodeAndOptionalOp(trace1Output); node && loadOp) { - if (is(trace1.Type())) + if (is(trace1Output.Type())) { // Map the memory state output to the corresponding memory state input - auto & memoryStateInput = LoadOperation::MapMemoryStateOutputToInput(trace1); - return *memoryStateInput.origin(); + auto & memoryStateInput = LoadOperation::MapMemoryStateOutputToInput(trace1Output); + return TraceStepResult::createStepResult(*memoryStateInput.origin()); } } } + // We were not able to make any extra progress on the trace result, so return it as is return trace1; } @@ -66,6 +72,7 @@ traceOutput(rvsdg::Output & output, bool mayEnterSubregions, const rvsdg::Region mayEnterSubregions ? rvsdg::OutputTracer::StructuralNodePolicy::traceIntoSubregions : rvsdg::OutputTracer::StructuralNodePolicy::traceThroughIfDetectedInvariant); + tracer.setRegionPredicateCheckingEnabled(mayEnterSubregions); tracer.setEnterPhiNodes(mayEnterSubregions); return tracer.trace(output, withinRegion); } diff --git a/jlm/llvm/ir/Trace.hpp b/jlm/llvm/ir/Trace.hpp index c580207f1..d60deb6c7 100644 --- a/jlm/llvm/ir/Trace.hpp +++ b/jlm/llvm/ir/Trace.hpp @@ -42,8 +42,9 @@ class OutputTracer : public rvsdg::OutputTracer } protected: - [[nodiscard]] rvsdg::Output & - traceStep(rvsdg::Output & output, const rvsdg::Region * withinRegion) override; + [[nodiscard]] TraceStepResult + traceStep(rvsdg::Output & output, BackEdgeState backEdgeState, const rvsdg::Region * withinRegion) + override; private: bool traceThroughLoadedStates_ = false; diff --git a/jlm/llvm/opt/alias-analyses/AliasAnalysisPrecisionEvaluator.hpp b/jlm/llvm/opt/alias-analyses/AliasAnalysisPrecisionEvaluator.hpp index 574ed01b4..beacfda14 100644 --- a/jlm/llvm/opt/alias-analyses/AliasAnalysisPrecisionEvaluator.hpp +++ b/jlm/llvm/opt/alias-analyses/AliasAnalysisPrecisionEvaluator.hpp @@ -14,11 +14,6 @@ #include -namespace jlm::util -{ -class GraphWriter; -} - namespace jlm::llvm::aa { diff --git a/jlm/rvsdg/Trace.cpp b/jlm/rvsdg/Trace.cpp index 4f6138a7f..1a5246fcd 100644 --- a/jlm/rvsdg/Trace.cpp +++ b/jlm/rvsdg/Trace.cpp @@ -24,19 +24,47 @@ OutputTracer::trace(Output & output) } Output & -OutputTracer::trace(Output & output, const rvsdg::Region * withinRegion) +OutputTracer::trace(Output & output, const Region * withinRegion) +{ + // FIXME(perf): Querying the region predication checker class causes repeated re-traversal of + // the same paths through the region hierarchy that the tracer is already taking. + // Performance could be improved by adding a way of extracting impossible regions directy. + regionPredicateTracer_.clearCaches(); + + // Mark this as the starting output, which becomes the target for region predicate reachability + startingOutput_ = &output; + // Since the current output is the starting output, no back-edges have been followed + BackEdgeState backEdgeState = BackEdgeState::NoBackEdgeTaken; + + // To disable region predicate checking, always assume back-edges have been taken + if (!isRegionPredicateCheckingEnabled()) + backEdgeState = BackEdgeState::PossiblyBackEdgeTaken; + + return traceInternal(output, backEdgeState, withinRegion); +} + +Output & +OutputTracer::traceInternal( + Output & output, + BackEdgeState backEdgeState, + const Region * withinRegion) { Output * head = &output; - // Keep tracing until the output stops changing + // Keep tracing until a final result is reached while (true) { - Output * prevHead = head; - head = &traceStep(*head, withinRegion); - if (head == prevHead) - { - return *head; - } + const auto traceStepResult = traceStep(*head, backEdgeState, withinRegion); + + // If the tracing step is final, we are done + if (traceStepResult.isFinalResult()) + return traceStepResult.getOutput(); + + // If the tracing step is not final, it must have made progress + JLM_ASSERT(&traceStepResult.getOutput() != head); + + // Keep tracing from the step result + head = &traceStepResult.getOutput(); } } @@ -52,64 +80,129 @@ mapGammaArgumentToOrigin(GammaNode & gammaNode, Output & output) return *gammaNode.mapBranchArgumentToInput(output).origin(); } -Output * -OutputTracer::tryTraceThroughGamma(GammaNode & gammaNode, Output & output) +OutputTracer::TraceStepResult +OutputTracer::traceGammaOutput(GammaNode & gammaNode, Output & output, BackEdgeState backEdgeState) { + // First check the invariance cache if (const auto invariantValueInput = lookupInInvarianceCache(output)) { - return invariantValueInput->origin(); + return TraceStepResult::createStepResult(*invariantValueInput->origin()); } const auto exitVar = gammaNode.MapOutputExitVar(output); // The shared output that is the origin of the entry variable(s) going into the gamma node - Output * commonOrigin = nullptr; - Input * gammaInput = nullptr; + // nullopt means no subregion has been traced yet. + // nullptr means there can be no shared common outer origin. + std::optional commonOuterOrigin; + // The gamma input that gets its value from the common outer origin + Input * commonGammaInput = nullptr; + + // If only a single subregion can provide the value, this is the origin within that subregion. + // nullopt means no valid subregion been found yet. + // nullptr means there are multiple valid subregions. + std::optional singleInnerOrigin; + if (!isTracingIntoSubregionsEnabled()) + { + singleInnerOrigin = nullptr; + } for (auto branchResult : exitVar.branchResult) { + // Region predication checking requires that no back-edge has been taken around the gamma + if (backEdgeState == BackEdgeState::NoBackEdgeTaken) + { + // If control flow can not go from the gamma subregion to the region of the starting output, + // it can not be the origin of the traced value. + if (!regionPredicateTracer_.isReachableFromRegion( + *startingOutput_->region(), + *branchResult->region())) + continue; + } + auto tracedInner = branchResult->origin(); if (isDeepInvarianceCheckingEnabled()) { // Trace the branch result origin, but only within the gamma subregion - tracedInner = &trace(*tracedInner, tracedInner->region()); + tracedInner = &traceInternal(*tracedInner, backEdgeState, tracedInner->region()); } - // TODO: handle the traceIntoSubregion policy as well, - // when only one of the gamma subregions provides a defined and reachable value, - // that value can be returned - - // The traced output must reach a region argument in the gamma subregion - if (TryGetRegionParentNode(*tracedInner) != &gammaNode) - return nullptr; - - // Get the origin of the region argument outside the gamma - gammaInput = &gammaNode.mapBranchArgumentToInput(*tracedInner); - Output & outerOrigin = *gammaInput->origin(); + // Set the single inner origin, or clear it if we already had one + if (!singleInnerOrigin.has_value()) + { + singleInnerOrigin = tracedInner; + } + else + { + singleInnerOrigin = nullptr; + } - // Check that the origin matches with all other origins - if (commonOrigin == nullptr) + // Check if the subregion result was traced all the way to an argument + if (TryGetRegionParentNode(*tracedInner) == &gammaNode) + { + // Get the origin of the region argument outside the gamma + auto & gammaInput = gammaNode.mapBranchArgumentToInput(*tracedInner); + Output & outerOrigin = *gammaInput.origin(); + + // If this is the first outer origin, make it the common outer origin for now + if (!commonOuterOrigin.has_value()) + { + commonOuterOrigin = &outerOrigin; + commonGammaInput = &gammaInput; + } + else if (*commonOuterOrigin != &outerOrigin) + { + // Mismatching outer origins found, given up on finding a common outer origin + commonOuterOrigin = nullptr; + } + } + else { - commonOrigin = &outerOrigin; + // The subregion result could not be traced to an outer origin + commonOuterOrigin = nullptr; } - else if (commonOrigin != &outerOrigin) + + // Stop looping through subregions if there is neither an inner origin or a common outer origin + if (commonOuterOrigin == nullptr && singleInnerOrigin == nullptr) + return TraceStepResult::createFinalResult(output); + } + + // If we found a common outer origin, continue tracing from there + if (commonOuterOrigin.has_value() && *commonOuterOrigin != nullptr) + { + JLM_ASSERT(commonGammaInput != nullptr); + + // If the gamma was invariant, even with no assumptions about back-edges not being taken + // around the gamma, the invariance can be added to the cache + if (backEdgeState == BackEdgeState::PossiblyBackEdgeTaken) { - return nullptr; + insertInInvarianceCache(output, *commonGammaInput); } + + return TraceStepResult::createStepResult(*commonGammaInput->origin()); + } + + // If only a single gamma subregion provides a possible origin, use it + if (singleInnerOrigin.has_value() && *singleInnerOrigin != nullptr) + { + JLM_ASSERT(isTracingIntoSubregionsEnabled()); + + // The origins found inside subregions have already been fully traced, so they are final + return TraceStepResult::createFinalResult(**singleInnerOrigin); } - JLM_ASSERT(commonOrigin != nullptr); - JLM_ASSERT(gammaInput != nullptr); - return insertInInvarianceCache(output, *gammaInput); + // Tracing was unable to make any progress beyond the gamma output + return TraceStepResult::createFinalResult(output); } -Output * -OutputTracer::traceThetaOutput(ThetaNode & thetaNode, Output & output) +OutputTracer::TraceStepResult +OutputTracer::traceThetaOutput(ThetaNode & thetaNode, Output & output, BackEdgeState backEdgeState) { + // Lookup the output in the invariance cache if (const auto invariantValueInput = lookupInInvarianceCache(output)) { - return invariantValueInput->origin(); + return TraceStepResult::createStepResult(*invariantValueInput->origin()); } const auto loopVar = thetaNode.MapOutputLoopVar(output); @@ -120,23 +213,48 @@ OutputTracer::traceThetaOutput(ThetaNode & thetaNode, Output & output) if (isDeepInvarianceCheckingEnabled()) { // trace the origin within the thetaNode, but only within the theta's subregion - tracedInner = &trace(*tracedInner, thetaNode.subregion()); + tracedInner = &traceInternal(*tracedInner, backEdgeState, thetaNode.subregion()); } - // If tracing reached the pre argument of the same loop variable, it is invariant + // If tracing reached the pre argument of the same loop variable, it might be invariant if (tracedInner == loopVar.pre) { - return insertInInvarianceCache(output, *loopVar.input); - } + // If the loop variable was found to be invariant, + // but we also made an assumption about not taking any back-edges around the theta subregion, + // we must check again without making that assumption to be sure it is acutually invariant. + + if (backEdgeState == BackEdgeState::PossiblyBackEdgeTaken) + { + // The tracing already made no assumptions about back-edges. + // The loop variable is definitely invariant + return TraceStepResult::createStepResult(insertInInvarianceCache(output, *loopVar.input)); + } + + // Try tracing from the loop var post again, this time with no assumption + auto tracedInnerAgain = &traceInternal( + *loopVar.post->origin(), + BackEdgeState::PossiblyBackEdgeTaken, + thetaNode.subregion()); + if (tracedInnerAgain == loopVar.pre) + { + // The loop variable is in fact invariant, connect the output to the loop variable input + return TraceStepResult::createStepResult(insertInInvarianceCache(output, *loopVar.input)); + } - // If tracing from the post result lead to the pre argument of a different loop variable, - // check if that loop variable is trivially invariant, and if it is, return its input origin. - if (TryGetRegionParentNode(*tracedInner) == &thetaNode) + // If we get here, it means that the loop variable was only found to be invariant in the final + // iteration of the loop, but not in every iteration + JLM_ASSERT(!rvsdg::ThetaLoopVarIsInvariant(loopVar)); + } + else if (TryGetRegionParentNode(*tracedInner) == &thetaNode) { + // Tracing from the post result lead to the pre argument of a different loop variable. + // Check if that loop variable is trivially invariant, and if it is, return its input origin. + auto originLoopVar = thetaNode.MapPreLoopVar(*tracedInner); if (ThetaLoopVarIsInvariant(originLoopVar)) { - return insertInInvarianceCache(output, *originLoopVar.input); + return TraceStepResult::createStepResult( + insertInInvarianceCache(output, *originLoopVar.input)); } } @@ -144,87 +262,83 @@ OutputTracer::traceThetaOutput(ThetaNode & thetaNode, Output & output) // return the result from tracing inside the subregion if (isTracingIntoSubregionsEnabled()) { - return tracedInner; + // The origin found inside the theta is already fully traced, so it is final + return TraceStepResult::createFinalResult(*tracedInner); } // Otherwise, we are unable to trace further from the theta output - return nullptr; + return TraceStepResult::createFinalResult(output); } -Output & -OutputTracer::traceStep(Output & output, const rvsdg::Region * withinRegion) +OutputTracer::TraceStepResult +OutputTracer::traceThetaArgument(ThetaNode & thetaNode, Output & output) +{ + // Get the loop variable + auto loopVar = thetaNode.MapPreLoopVar(output); + + // Trace from the corresponding theta output by following the back-edge + auto tracedOutput = + traceThetaOutput(thetaNode, *loopVar.output, BackEdgeState::PossiblyBackEdgeTaken); + + // If the loop output is invariant and has the same origin as the loop variable, + // tracing can continue from outside the theta + if (&tracedOutput.getOutput() == loopVar.input->origin()) + { + JLM_ASSERT(!tracedOutput.isFinalResult()); + return TraceStepResult::createStepResult(tracedOutput.getOutput()); + } + + // Otherwise tracing stops at the theta argument + return TraceStepResult::createFinalResult(output); +} + +OutputTracer::TraceStepResult +OutputTracer::traceStep(Output & output, BackEdgeState backEdgeState, const Region * withinRegion) { if (withinRegion && withinRegion == TryGetOwnerRegion(output)) { // We are not allowed to leave this region, and tracing has reached one of its arguments - return output; + return TraceStepResult::createFinalResult(output); } // Handle gamma node outputs if (const auto gammaNode = TryGetOwnerNode(output)) { - if (const auto traced = tryTraceThroughGamma(*gammaNode, output)) - return *traced; - - return output; + return traceGammaOutput(*gammaNode, output, backEdgeState); } // Handle gamma node arguments if (const auto gammaNode = TryGetRegionParentNode(output)) { - return mapGammaArgumentToOrigin(*gammaNode, output); + return TraceStepResult::createStepResult(mapGammaArgumentToOrigin(*gammaNode, output)); } // Handle theta node outputs if (const auto thetaNode = TryGetOwnerNode(output)) { - if (const auto traced = traceThetaOutput(*thetaNode, output)) - { - return *traced; - } - - return output; + return traceThetaOutput(*thetaNode, output, backEdgeState); } // Handle theta node arguments if (const auto thetaNode = TryGetRegionParentNode(output)) { - // Tracing from inside a theta to outside it is only valid if the loop variable is invariant. - // This is determined by tracing from the loop variable's post result, - // and seeing if it leads to the same input origin as the loop variable's own input. - - // The loop variable whose pre argument is being traced from - const auto loopVar = thetaNode->MapPreLoopVar(output); - - // The origin of the loop variable's input. - const auto inputOrigin = loopVar.input->origin(); - - // The origin reached when tracing from the loop variable's post result, - // if it reaches an invariant loop variable and "escapes" the theta. - // The invariant loop variable found does not have to be the same as the above loopVar. - // See TraceTests' TestIndirectLoopInvariance. - const auto postOrigin = traceThetaOutput(*thetaNode, *loopVar.output); - - if (postOrigin == inputOrigin) - { - return *inputOrigin; - } - - return output; + // The backEdgeState is not provided to this function, + // since it must anyways immediately follow a back-edge to determine loop invariance + return traceThetaArgument(*thetaNode, output); } // If we are not doing interprocedural tracing, stop tracing now if (!isInterprocedural_) - return output; + return TraceStepResult::createFinalResult(output); // Handle lambda context variables if (const auto lambda = TryGetRegionParentNode(output)) { // If the argument is a contex variable, continue tracing if (const auto ctxVar = lambda->MapBinderContextVar(output)) - return *ctxVar->input->origin(); + return TraceStepResult::createStepResult(*ctxVar->input->origin()); - return output; + return TraceStepResult::createFinalResult(output); } // Handle delta context variables @@ -232,7 +346,7 @@ OutputTracer::traceStep(Output & output, const rvsdg::Region * withinRegion) { // If the argument is a contex variable, continue tracing const auto ctxVar = delta->MapBinderContextVar(output); - return *ctxVar.input->origin(); + return TraceStepResult::createStepResult(*ctxVar.input->origin()); } // Handle phi outputs @@ -241,9 +355,9 @@ OutputTracer::traceStep(Output & output, const rvsdg::Region * withinRegion) if (enterPhiNodes_) { const auto fixVar = phiNode->MapOutputFixVar(output); - return *fixVar.result->origin(); + return TraceStepResult::createStepResult(*fixVar.result->origin()); } - return output; + return TraceStepResult::createFinalResult(output); } // Handle phi region arguments @@ -256,24 +370,24 @@ OutputTracer::traceStep(Output & output, const rvsdg::Region * withinRegion) if (const auto ctxVar = std::get_if(&argument)) { // Follow the context variable to outside the phi - return *ctxVar->input->origin(); + return TraceStepResult::createStepResult(*ctxVar->input->origin()); } - return output; + return TraceStepResult::createFinalResult(output); } - return output; + return TraceStepResult::createFinalResult(output); } -Output * +Output & OutputTracer::insertInInvarianceCache(const Output & output, Input & traceResult) { if (enableInvarianceCaching_) { - JLM_ASSERT(invariantOutputCache_.find(&output) == invariantOutputCache_.end()); - invariantOutputCache_[&output] = &traceResult; + const auto [_, inserted] = invariantOutputCache_.emplace(&output, &traceResult); + JLM_ASSERT(inserted); } - return traceResult.origin(); + return *traceResult.origin(); } Input * @@ -298,16 +412,18 @@ traceOutputIntraProcedurally(Output & output, bool mayEnterSubregions) tracer.setStructuralNodePolicy( mayEnterSubregions ? OutputTracer::StructuralNodePolicy::traceIntoSubregions : OutputTracer::StructuralNodePolicy::traceThroughIfDetectedInvariant); + tracer.setRegionPredicateCheckingEnabled(mayEnterSubregions); return tracer.trace(output); } Output & -traceOutput(Output & output, bool mayEnterSubregions, const rvsdg::Region * withinRegion) +traceOutput(Output & output, bool mayEnterSubregions, const Region * withinRegion) { OutputTracer tracer; tracer.setStructuralNodePolicy( mayEnterSubregions ? OutputTracer::StructuralNodePolicy::traceIntoSubregions : OutputTracer::StructuralNodePolicy::traceThroughIfDetectedInvariant); + tracer.setRegionPredicateCheckingEnabled(mayEnterSubregions); tracer.setEnterPhiNodes(mayEnterSubregions); return tracer.trace(output, withinRegion); } diff --git a/jlm/rvsdg/Trace.hpp b/jlm/rvsdg/Trace.hpp index eb62f3ced..5da55b1cf 100644 --- a/jlm/rvsdg/Trace.hpp +++ b/jlm/rvsdg/Trace.hpp @@ -7,6 +7,7 @@ #define JLM_RVSDG_TRACE_HPP #include +#include namespace jlm::rvsdg { @@ -154,6 +155,18 @@ class OutputTracer enterPhiNodes_ = value; } + [[nodiscard]] bool + isRegionPredicateCheckingEnabled() const noexcept + { + return enableRegionPredicateChecking_; + } + + void + setRegionPredicateCheckingEnabled(bool value) noexcept + { + enableRegionPredicateChecking_ = value; + } + /** * Controls whether caching of structural output invariance is enabled, * which can speed up tracing through deeply nested graphs. @@ -217,49 +230,155 @@ class OutputTracer * @param withinRegion the region where we stop tracing. */ [[nodiscard]] Output & - trace(Output & output, const rvsdg::Region * withinRegion); + trace(Output & output, const Region * withinRegion); protected: + // Enum representing information about the path the tracer took from the starting output + // to reach the current output being considered + enum class BackEdgeState + { + // Tracing has gone from the starting output to the current output without + // following any back-edges around the current output. + // Theta nodes between the current output and the starting output do not matter. + NoBackEdgeTaken, + + // While tracing from the starting output to the current output, + // the tracer may have followed a back-edge going around the current output. + // This prevents the use of the region predication checker. + PossiblyBackEdgeTaken, + }; + /** - * Attempts to trace the output of a gamma node through the node. - * This is only possible if the output can be traced to a gamma entry variable in all subregions, - * and these entry variables all share the same origin outside the gamma. + * Internal class used for returning intermediate results during tracing. + */ + class TraceStepResult + { + public: + /** + * The output reached after tracing zero, one or more steps. + * + * @return the output arrived at by the tracing function + */ + [[nodiscard]] Output & + getOutput() const noexcept + { + return output_; + } + + /** + * Indicates whether the returned output can be traced any further, or if tracing is done. + * When done, the caller should not attempt any further tracing from the resulting output. + * + * @return true if the result is final, false if further tracing might be possible + */ + [[nodiscard]] bool + isFinalResult() const noexcept + { + return isFinalResult_; + } + + /** + * Creates an instance representing a non-final tracing result, + * that can possibly be traced further. + */ + [[nodiscard]] static TraceStepResult + createStepResult(Output & output) + { + return TraceStepResult(output, false); + } + + /** + * Creates an instance representing a final tracing result, + * from which further tracing is not possible. + */ + [[nodiscard]] static TraceStepResult + createFinalResult(Output & output) + { + return TraceStepResult(output, true); + } + + private: + TraceStepResult(Output & output, bool isFinalResult) + : output_(output), + isFinalResult_(isFinalResult) + {} + + Output & output_; + bool isFinalResult_; + }; + + /** + * Performs tracing from the given \p output, without updating the current starting output. + * Keeps tracing until the tracer is unable to find a more canonical output, + * or until an argument of the optional \p withinRegion limit is reached. + * + * @param output the output to trace from. + * @param backEdgeState enum describing the path taken from the starting output to \p output. + * @param withinRegion the region tracing has to stay within, or nullptr + * @return the resulting output reached when no more tracing is possible + */ + [[nodiscard]] Output & + traceInternal(Output & output, BackEdgeState backEdgeState, const Region * withinRegion); + + /** + * Trace from the given gamma output. + * If the gamma output can be traced to the same gamma input in all subregions, + * the origin of the gamma input is returned. + * Otherwise, if the tracer is allowed to trace into subregions, + * and only one of the subregions can be the origin of the value being traced, + * the origin within that subregion is returned. + * Otherwise, the gamma output itself is returned. * * @pre the \p output is an output of the given \p gammaNode * * @param gammaNode the gamma node to trace through * @param output an output of the given gamma node - * @return the origin of the output value on the input side of the gamma, or nullptr. + * @param backEdgeState enum describing the path taken from the starting output to \p output. + * @return the result of tracing from the gamma output */ - [[nodiscard]] Output * - tryTraceThroughGamma(GammaNode & gammaNode, Output & output); + [[nodiscard]] TraceStepResult + traceGammaOutput(GammaNode & gammaNode, Output & output, BackEdgeState backEdgeState); /** * Trace from the the given loop output. * If the loop output is found to be loop invariant, the origin of the theta input is returned. * Otherwise, if the tracer is allowed to trace into the subregion of structural nodes, * the traced origin of the loop variable post inside the theta is returned. - * Otherwise, nullopt is returned. + * Otherwise, the theta output itself is returned. * * @pre the \p output is an output of the given \p thetaNode * * @param thetaNode the theta node to trace through * @param output an output of the given theta node - * @return the origin of the output value on the input side of the theta, - * the origin of the loop variable post, or nullopt + * @param backEdgeState enum describing the path taken from the starting output to \p output. + * @return the result of tracing from the theta output */ - [[nodiscard]] Output * - traceThetaOutput(ThetaNode & thetaNode, Output & output); + [[nodiscard]] TraceStepResult + traceThetaOutput(ThetaNode & thetaNode, Output & output, BackEdgeState backEdgeState); + + /** + * Trace from the given loop variable pre argument. + * This function is only used when the tracing starting point is inside the theta. + * + * @pre the \p output is an argument of the given \p thetaNode's subregion. + * + * @param thetaNode the theta node to attempt to trace out of + * @param output the theta subregion argument + * @return the result of tracing from the theta argument + */ + [[nodiscard]] TraceStepResult + traceThetaArgument(ThetaNode & thetaNode, Output & output); /** * The innermost body of the tracing loop. Should trace at least one step, if possible. * If it is not possible to trace further, the same output is returned. * @param output the output to trace from. + * @param backEdgeState enum describing the path taken from the starting output to \p output. * @param withinRegion if not nullptr, tracing stops if it reaches an argument of the region. - * @return the result of tracing from the given output, if possible. Otherwise, \p output. + * @return the result of tracing from the given output */ - [[nodiscard]] virtual Output & - traceStep(Output & output, const rvsdg::Region * withinRegion); + [[nodiscard]] virtual TraceStepResult + traceStep(Output & output, BackEdgeState backEdgeState, const Region * withinRegion); /** * Inserts the given \p structuralOutput in the invariance cache. @@ -267,11 +386,14 @@ class OutputTracer * of the structural node, so tracing can pass through the structural node without * looking inside its subregions. * + * When determining invariance, it is important to not makse assumptions about + * tracing never having followed back-edges. + * * @param structuralOutput The structural output that was traced. * @param structuralInput The corresponding structural input. * @return The origin of \p structuralInput for convenience. */ - Output * + Output & insertInInvarianceCache(const Output & structuralOutput, Input & structuralInput); /** @@ -295,10 +417,22 @@ class OutputTracer // When false, tracing will stop at the output of the phi node. bool enterPhiNodes_ = true; + // When true, gamma subregions are ignored when it is impossible for control flow to go + // from the gamma subregion to the region containing the output tracing started from + bool enableRegionPredicateChecking_ = false; + // The region predicate checker used to disqualify regions + AlternativeRegionPredicateTracer regionPredicateTracer_; + // The output from which the current tracing operation started. + // Used for region predication checks. + // This is the starting output referenced by the enum \ref BackEdgeState. + const Output * startingOutput_ = nullptr; + // When true, the tracer can cache the fact that outputs of structural nodes are invariant. // Enabling caching means the user of the tracer is responsible for cache invalidation. // @see clearInvarianceCache() for details bool enableInvarianceCaching_ = false; + // Maps from a structural output to an input of the same structural node + // that the output always gets its value from. std::unordered_map invariantOutputCache_{}; }; @@ -348,13 +482,10 @@ traceOutputIntraProcedurally(const Output & output, bool mayEnterSubregions) * @return the final value of the tracing */ Output & -traceOutput(Output & output, bool mayEnterSubregions, const rvsdg::Region * withinRegion = nullptr); +traceOutput(Output & output, bool mayEnterSubregions, const Region * withinRegion = nullptr); inline const Output & -traceOutput( - const Output & output, - bool mayEnterSubregions, - const rvsdg::Region * withinRegion = nullptr) +traceOutput(const Output & output, bool mayEnterSubregions, const Region * withinRegion = nullptr) { return traceOutput(const_cast(output), mayEnterSubregions, withinRegion); } diff --git a/jlm/rvsdg/TraceTests.cpp b/jlm/rvsdg/TraceTests.cpp index 9a5c07903..7096923b9 100644 --- a/jlm/rvsdg/TraceTests.cpp +++ b/jlm/rvsdg/TraceTests.cpp @@ -411,6 +411,8 @@ TEST(TraceTests, GammaCachingTest) OutputTracer tracer; tracer.setInvarianceCaching(true); + // predicate checking reduces caching of gamma nodes, so disable it for this test + tracer.setRegionPredicateCheckingEnabled(false); // Act & Assert // This is the first time we are tracing this output. We expect it to arrive at i1. @@ -473,3 +475,213 @@ TEST(TraceTests, ThetaCachingTest) traceResult = &tracer.trace(*graphExport.origin()); assert(traceResult == loopVar1.output); } + +TEST(TraceTests, RegionPredicationThetaTest) +{ + using namespace jlm::rvsdg; + + /** + * Creates an RVSDG that looks like + * + * Int(1) Int(2) Int(3) + * v v v + * +-theta-------x-------x-------x-------------------+ + * | | | | | + * | TestOp | | | | + * | v v v v | + * | +-gamma---x--x--x------+---------x--x------x-+ | + * | | | | | | | | | + * | | CTRL(0) | | Int(4) | CTRL(1) | Int(5) | | | + * | | v v v v | v v v v | | + * | +---x-----x--x--x------+---x-----x----x----x-+ | + * | | | | | | + * | v v v v | + * +-----x-------x-------x-------x-------------------+ + * | | | + * v v v + * exp(x) exp(y) exp(z) + * + * and checks that tracing from "x" leads all the way to Int(1), + * tracing from "y" leads to to the pre of the second loop variable, + * and tracing from "z" leads to Int(4) + */ + + // Arrange + const auto controlType = ControlType::Create(2); + const auto int32Type = BitType::Create(32); + + Graph rvsdg; + + auto & int1Outer = BitConstantOperation::create(rvsdg.GetRootRegion(), { 32, 1 }); + auto & int2Outer = BitConstantOperation::create(rvsdg.GetRootRegion(), { 32, 2 }); + auto & int3Outer = BitConstantOperation::create(rvsdg.GetRootRegion(), { 32, 3 }); + + auto & thetaNode = *ThetaNode::create(&rvsdg.GetRootRegion()); + auto loopVar1 = thetaNode.AddLoopVar(&int1Outer); + auto loopVar2 = thetaNode.AddLoopVar(&int2Outer); + auto loopVar3 = thetaNode.AddLoopVar(&int3Outer); + + auto & testOp = + jlm::rvsdg::CreateOpNode(*thetaNode.subregion(), controlType); + + auto & gammaNode = *GammaNode::create(testOp.output(0), 2); + auto gammaEntry1 = gammaNode.AddEntryVar(loopVar1.pre); + auto gammaEntry2 = gammaNode.AddEntryVar(loopVar2.pre); + auto gammaEntry3 = gammaNode.AddEntryVar(loopVar3.pre); + + // left subregion + auto & gammaCtrl0 = ControlConstantOperation::createFalse(*gammaNode.subregion(0)); + auto & gammaInt4 = BitConstantOperation::create(*gammaNode.subregion(0), { 32, 4 }); + + // right subregion + auto & gammaCtrl1 = ControlConstantOperation::createTrue(*gammaNode.subregion(1)); + auto & gammaInt5 = BitConstantOperation::create(*gammaNode.subregion(1), { 32, 5 }); + + auto gammaCtrlExit = gammaNode.AddExitVar({ &gammaCtrl0, &gammaCtrl1 }); + auto gammaExit1 = + gammaNode.AddExitVar({ gammaEntry1.branchArgument[0], gammaEntry1.branchArgument[1] }); + auto gammaExit2 = gammaNode.AddExitVar({ gammaEntry2.branchArgument[0], &gammaInt5 }); + auto gammaExit3 = gammaNode.AddExitVar({ &gammaInt4, gammaEntry3.branchArgument[1] }); + + // Divert theta results + thetaNode.predicate()->divert_to(gammaCtrlExit.output); + loopVar1.post->divert_to(gammaExit1.output); + loopVar2.post->divert_to(gammaExit2.output); + loopVar3.post->divert_to(gammaExit3.output); + + // exports + auto & exportX = GraphExport::Create(*loopVar1.output, "x"); + auto & exportY = GraphExport::Create(*loopVar2.output, "y"); + auto & exportZ = GraphExport::Create(*loopVar3.output, "z"); + + // Act + OutputTracer tracer; + // Enable caching to check that the inner gamma is not incorrectly cached when not invariant + tracer.setInvarianceCaching(true); + tracer.setRegionPredicateCheckingEnabled(true); + tracer.setStructuralNodePolicy(OutputTracer::StructuralNodePolicy::traceIntoSubregions); + + // Assert + + // Tracing from x leads to the theta's input + ASSERT_EQ(&tracer.trace(*exportX.origin()), &int1Outer); + + // Tracing from y leads to the loop variable pre, + // since the loop variable is invariant in the last loop iteration, + // but not in any other loop iterations + ASSERT_EQ(&tracer.trace(*exportY.origin()), loopVar2.pre); + + // Tracing from z leads to the Int(4), + // since the last iteration always goes through the left subregion + ASSERT_EQ(&tracer.trace(*exportZ.origin()), &gammaInt4); + + // Also try tracing from the gamma exits inside the theta + // gamma exit 1 can be traced all the way out of the theta + ASSERT_EQ(&tracer.trace(*gammaExit1.output), &int1Outer); + + // gamma exit 2 and 3 can not be traced any further + ASSERT_EQ(&tracer.trace(*gammaExit2.output), gammaExit2.output); + ASSERT_EQ(&tracer.trace(*gammaExit3.output), gammaExit3.output); + + // Among the theta pre variables, only loop variable 1 can be traced anywhere + ASSERT_EQ(&tracer.trace(*loopVar1.pre), &int1Outer); + ASSERT_EQ(&tracer.trace(*loopVar2.pre), loopVar2.pre); + ASSERT_EQ(&tracer.trace(*loopVar3.pre), loopVar3.pre); +} + +TEST(TraceTests, RegionPredicationThetaToGammaTest) +{ + using namespace jlm::rvsdg; + + /** + * Creates an RVSDG that looks like + * + * TestOp TestOp + * v v + * +-theta-------x-------x--------------------------------------------------------+ + * | | + * | TestOp | + * | v | + * | +-gamma------------------+------------------------+------------------------+ | + * | | | | | | + * | | CTRL(0) CTRL(0) Int(1) | CTRL(0) CTRL(1) Int(2) | CTRL(1) CTRL(0) Int(3) | | + * | | v v v | v v v | v v v | | + * | +---x-------x-------x----+---x-------x-------x----+---x-------x-------x----+ | + * | | | | | + * | v v v | + * +-----x-------x-------x--------------------------------------------------------+ + * | | + * v v + * +-gamma-------x----+----x----+ + * | v | v | + * | here | here | + * +------------------+---------+ + * + * And traces from the entry variable arguments in the subregions of the second gamma. + * In the left subregion, tracing should lead back to Int(1), + * while the right subregion should lead back to Int(2). + */ + + // Arrange + const auto control2Type = ControlType::Create(2); + const auto control3Type = ControlType::Create(3); + const auto int32Type = BitType::Create(32); + + Graph rvsdg; + + auto & testOpCtrlOuter = + jlm::rvsdg::CreateOpNode(rvsdg.GetRootRegion(), control2Type); + auto & testOpIntOuter = + jlm::rvsdg::CreateOpNode(rvsdg.GetRootRegion(), int32Type); + + auto & thetaNode = *ThetaNode::create(&rvsdg.GetRootRegion()); + auto loopVarCtrl = thetaNode.AddLoopVar(testOpCtrlOuter.output(0)); + auto loopVarInt = thetaNode.AddLoopVar(testOpIntOuter.output(0)); + + auto & testOpCtrlInner = + jlm::rvsdg::CreateOpNode(*thetaNode.subregion(), control3Type); + auto & gammaInner = *GammaNode::create(testOpCtrlInner.output(0), 3); + + // subregion 0 + auto & sub0ctrl0 = ControlConstantOperation::createFalse(*gammaInner.subregion(0)); + auto & sub0Int1 = BitConstantOperation::create(*gammaInner.subregion(0), { 32, 1 }); + + // subregion 1 + auto & sub1ctrl0 = ControlConstantOperation::createFalse(*gammaInner.subregion(1)); + auto & sub1ctrl1 = ControlConstantOperation::createTrue(*gammaInner.subregion(1)); + auto & sub1Int2 = BitConstantOperation::create(*gammaInner.subregion(1), { 32, 2 }); + + // subregion 2 + auto & sub2ctrl0 = ControlConstantOperation::createFalse(*gammaInner.subregion(2)); + auto & sub2ctrl1 = ControlConstantOperation::createTrue(*gammaInner.subregion(2)); + auto & sub2Int3 = BitConstantOperation::create(*gammaInner.subregion(2), { 32, 3 }); + + // add exit variables + auto thetaPredExit = gammaInner.AddExitVar({ &sub0ctrl0, &sub1ctrl0, &sub2ctrl1 }); + auto innerToOuterGammaCtrlExit = gammaInner.AddExitVar({ &sub0ctrl0, &sub1ctrl1, &sub2ctrl0 }); + auto innerIntExit = gammaInner.AddExitVar({ &sub0Int1, &sub1Int2, &sub2Int3 }); + + // divert theta results + thetaNode.predicate()->divert_to(thetaPredExit.output); + loopVarCtrl.post->divert_to(innerToOuterGammaCtrlExit.output); + loopVarInt.post->divert_to(innerIntExit.output); + + // create outer gamma + auto & outerGamma = *GammaNode::create(loopVarCtrl.output, 2); + auto outerIntEntry = outerGamma.AddEntryVar(loopVarInt.output); + + // Act + OutputTracer tracer; + // Enable caching to check that the inner gamma is not incorrectly cached when not invariant + tracer.setInvarianceCaching(true); + tracer.setRegionPredicateCheckingEnabled(true); + tracer.setStructuralNodePolicy(OutputTracer::StructuralNodePolicy::traceIntoSubregions); + + // Assert + // Going from the subregions of the outer gamma leads to specific values in the inner gamma + ASSERT_EQ(&tracer.trace(*outerIntEntry.branchArgument[0]), &sub0Int1); + ASSERT_EQ(&tracer.trace(*outerIntEntry.branchArgument[1]), &sub1Int2); + + // Tracing directly from the theta output leads to the output of the inner gamma + ASSERT_EQ(&tracer.trace(*loopVarInt.output), innerIntExit.output); +}