stmt_functor.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#ifndef TVM_TIRX_STMT_FUNCTOR_H_
#define TVM_TIRX_STMT_FUNCTOR_H_
 
#include <tvm/ir/node_functor.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/function.h>
#include <tvm/tirx/stmt.h>
#include <tvm/tirx/tile_primitive.h>
 
#include <unordered_map>
#include <utility>
 
namespace tvm {
namespace tirx {
template <typename FType>
class StmtFunctor;
 
#define STMT_FUNCTOR_DEFAULT                                   \
  {                                                            \
    return VisitStmtDefault_(op, std::forward<Args>(args)...); \
  }
 
#define IR_STMT_FUNCTOR_DISPATCH(OP)                                                        \
  vtable.template set_dispatch<OP>([](const ffi::ObjectRef& n, TSelf* self, Args... args) { \
    return self->VisitStmt_(static_cast<const OP*>(n.get()), std::forward<Args>(args)...);  \
  });
 
template <typename R, typename... Args>
class StmtFunctor<R(const Stmt& n, Args... args)> {
 private:
  using TSelf = StmtFunctor<R(const Stmt& n, Args... args)>;
  using FType = NodeFunctor<R(const ffi::ObjectRef& n, TSelf* self, Args... args)>;
 
 public:
  using result_type = R;
  virtual ~StmtFunctor() {}
  R operator()(const Stmt& n, Args... args) { return VisitStmt(n, std::forward<Args>(args)...); }
  virtual R VisitStmt(const Stmt& n, Args... args) {
    static FType vtable = InitVTable();
    return vtable(n, this, std::forward<Args>(args)...);
  }
  // Functions that can be overriden by subclass
  virtual R VisitStmt_(const BindNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const AttrStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const IfThenElseNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const ForNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const WhileNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const ReturnNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const BreakNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const ContinueNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const AllocBufferNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const DeclBufferNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const BufferStoreNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const AssertStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const SeqStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const EvaluateNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const SBlockNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const SBlockRealizeNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const ScopeIdDefStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmt_(const tirx::TilePrimitiveCallNode* op, Args... args) STMT_FUNCTOR_DEFAULT;
  virtual R VisitStmtDefault_(const ffi::Object* op, Args...) {
    TVM_FFI_THROW(InternalError) << "Do not have a default for " << op->GetTypeKey();
    TVM_FFI_UNREACHABLE();
  }
 
 private:
  // initialize the vtable.
  static FType InitVTable() {
    FType vtable;
    IR_STMT_FUNCTOR_DISPATCH(BindNode);
    IR_STMT_FUNCTOR_DISPATCH(AttrStmtNode);
    IR_STMT_FUNCTOR_DISPATCH(IfThenElseNode);
    IR_STMT_FUNCTOR_DISPATCH(ForNode);
    IR_STMT_FUNCTOR_DISPATCH(WhileNode);
    IR_STMT_FUNCTOR_DISPATCH(ReturnNode);
    IR_STMT_FUNCTOR_DISPATCH(BreakNode);
    IR_STMT_FUNCTOR_DISPATCH(ContinueNode);
    IR_STMT_FUNCTOR_DISPATCH(AllocBufferNode);
    IR_STMT_FUNCTOR_DISPATCH(DeclBufferNode);
    IR_STMT_FUNCTOR_DISPATCH(AssertStmtNode);
    IR_STMT_FUNCTOR_DISPATCH(SeqStmtNode);
    IR_STMT_FUNCTOR_DISPATCH(EvaluateNode);
    IR_STMT_FUNCTOR_DISPATCH(BufferStoreNode);
    IR_STMT_FUNCTOR_DISPATCH(SBlockNode);
    IR_STMT_FUNCTOR_DISPATCH(SBlockRealizeNode);
    IR_STMT_FUNCTOR_DISPATCH(ScopeIdDefStmtNode);
    IR_STMT_FUNCTOR_DISPATCH(tirx::TilePrimitiveCallNode);
    vtable.Finalize();
    return vtable;
  }
};
 
#undef IR_STMT_FUNCTOR_DISPATCH
#undef STMT_FUNCTOR_DEFAULT
 
class TVM_DLL StmtVisitor : protected StmtFunctor<void(const Stmt&)> {
 public:
  using StmtFunctor::operator();
 
 protected:
  using StmtFunctor::VisitStmt;
  virtual void VisitExpr(const Expr& e) {}
  virtual void VisitBufferDef(const BufferVar& buffer, bool alloc_data);
  virtual void VisitBufferUse(const BufferVar& buffer);
  // statement visitor
  void VisitStmt_(const BindNode* op) override;
  void VisitStmt_(const AttrStmtNode* op) override;
  void VisitStmt_(const IfThenElseNode* op) override;
  void VisitStmt_(const ForNode* op) override;
  void VisitStmt_(const WhileNode* op) override;
  void VisitStmt_(const ReturnNode* op) override;
  void VisitStmt_(const BreakNode* op) override;
  void VisitStmt_(const ContinueNode* op) override;
  void VisitStmt_(const AllocBufferNode* op) override;
  void VisitStmt_(const DeclBufferNode* op) override;
  void VisitStmt_(const BufferStoreNode* op) override;
  void VisitStmt_(const AssertStmtNode* op) override;
  void VisitStmt_(const SeqStmtNode* op) override;
  void VisitStmt_(const EvaluateNode* op) override;
  void VisitStmt_(const SBlockNode* op) override;
  void VisitStmt_(const SBlockRealizeNode* op) override;
  void VisitStmt_(const ScopeIdDefStmtNode* op) override;
  void VisitStmt_(const tirx::TilePrimitiveCallNode* op) override;
};
 
class TVM_DLL StmtMutator : protected StmtFunctor<Stmt(const Stmt&)> {
 public:
  Stmt operator()(Stmt stmt) {
    allow_copy_on_write_ = true;
    return VisitStmt(stmt);
  }
 
 protected:
  ffi::Map<BufferVar, BufferVar> buffer_remap_;
  // We perform copy on write optimizations on the StmtMutator
  // so that an unique copy of parent can be mutated inplace
  // when some of its children changed.
  // We only do such optimization for Stmt nests(instead of Exprs) for now
  // as Stmt's parent state is more likely remain unchanged when one of
  // its child block changes.
  bool allow_copy_on_write_{false};
  template <typename TNode>
  ffi::ObjectPtr<TNode> CopyOnWrite(const TNode* node) {
    static_assert(std::is_base_of<StmtNode, TNode>::value,
                  "StmtMutator:: CopyOnWrite requires us to track uniqueness of all parent "
                  "nodes during the recursion. Because the child classes do not necessarily "
                  "check the Array, Expr and other structures during the visit, it is only safe to "
                  "call this function with StmtNodes for now. "
                  "Please create a new node directly in other cases.");
    if (allow_copy_on_write_) {
      // return the old node.
      return ffi::GetObjectPtr<TNode>(const_cast<TNode*>(node));
    } else {
      // Make a new copy of the node.
      // need to rely on the default copy constructor
      return ffi::make_object<TNode>(*node);
    }
  }
  Stmt VisitStmt(const Stmt& stmt) override {
    if (allow_copy_on_write_ && !stmt.unique()) {
      allow_copy_on_write_ = false;
      Stmt ret = StmtFunctor::VisitStmt(stmt);
      allow_copy_on_write_ = true;
      return ret;
    } else {
      return StmtFunctor::VisitStmt(stmt);
    }
  }
  virtual Expr VisitExpr(const Expr& e) { return e; }
  PrimExpr VisitPrimExpr(const PrimExpr& e) { return VisitExpr(e).as_or_throw<PrimExpr>(); }
  virtual BufferVar VisitBufferDef(const BufferVar& buffer, bool alloc_data);
  virtual BufferVar VisitBufferUse(const BufferVar& buffer);
  // statement visitor
  Stmt VisitStmt_(const BindNode* op) override;
  Stmt VisitStmt_(const AttrStmtNode* op) override;
  Stmt VisitStmt_(const IfThenElseNode* op) override;
  Stmt VisitStmt_(const ForNode* op) override;
  Stmt VisitStmt_(const WhileNode* op) override;
  Stmt VisitStmt_(const ReturnNode* op) override;
  Stmt VisitStmt_(const BreakNode* op) override;
  Stmt VisitStmt_(const ContinueNode* op) override;
  Stmt VisitStmt_(const AllocBufferNode* op) override;
  Stmt VisitStmt_(const DeclBufferNode* op) override;
  Stmt VisitStmt_(const BufferStoreNode* op) override;
  Stmt VisitStmt_(const AssertStmtNode* op) override;
  Stmt VisitStmt_(const SeqStmtNode* op) override;
  Stmt VisitStmt_(const EvaluateNode* op) override;
  Stmt VisitStmt_(const SBlockNode* op) override;
  Stmt VisitStmt_(const SBlockRealizeNode* op) override;
  Stmt VisitStmt_(const ScopeIdDefStmtNode* op) override;
  Stmt VisitStmt_(const tirx::TilePrimitiveCallNode* op) override;
  Stmt VisitSeqStmt_(const SeqStmtNode* op, bool flatten_before_visit,
                     std::function<Stmt(const Stmt&)> fmutate = nullptr);
 
  // internal helper.
  class Internal;
};
 
class TVM_DLL StmtExprVisitor : public ExprVisitor, public StmtVisitor {
 public:
  using StmtVisitor::operator();
  using ExprVisitor::operator();
 
 protected:
  using ExprVisitor::VisitExpr;
  using ExprVisitor::VisitExpr_;
  using StmtVisitor::VisitStmt;
 
  void VisitExpr(const Expr& e) override { return ExprVisitor::VisitExpr(e); }
  void VisitExpr_(const TensorLoadNode* op) override;
  void VisitExpr_(const BufferRegionNode* op) override;
};
 
class TVM_DLL StmtExprMutator : public ExprMutator, public StmtMutator {
 public:
  using StmtMutator::operator();
  using ExprMutator::operator();
 
 protected:
  using ExprMutator::VisitExpr;
  using ExprMutator::VisitExpr_;
  using ExprMutator::VisitPrimExpr;
  using StmtMutator::VisitStmt;
 
  Expr VisitExpr(const Expr& e) override { return ExprMutator::VisitExpr(e); }
  Expr VisitExpr_(const VarNode* op) override;
  Expr VisitExpr_(const TensorLoadNode* op) override;
  Expr VisitExpr_(const BufferRegionNode* op) override;
};
 
TVM_DLL Stmt IRTransform(Stmt stmt, const ffi::Function& preorder, const ffi::Function& postorder,
                         ffi::Optional<ffi::Array<ffi::String>> only_enable = std::nullopt);
 
TVM_DLL void PostOrderVisit(const ffi::ObjectRef& node,
                            std::function<void(const ffi::ObjectRef&)> fvisit);
 
TVM_DLL Stmt Substitute(Stmt stmt, std::function<ffi::Optional<Expr>(const Var& var)> vmap);
 
TVM_DLL Expr Substitute(Expr expr, std::function<ffi::Optional<Expr>(const Var& var)> vmap);
 
inline PrimExpr Substitute(PrimExpr expr, std::function<ffi::Optional<Expr>(const Var& var)> vmap) {
  return Substitute(Expr(expr), std::move(vmap)).as_or_throw<PrimExpr>();
}
 
inline Range Substitute(const Range& range,
                        std::function<ffi::Optional<Expr>(const Var& var)> vmap) {
  return Range::FromMinExtent(Substitute(range->min, vmap), Substitute(range->extent, vmap));
}
 
template <typename T>
ffi::Array<T> Substitute(const ffi::Array<T>& arr,
                         std::function<ffi::Optional<Expr>(const Var& var)> vmap) {
  return arr.Map([&vmap](const auto& elem) { return Substitute(elem, vmap); });
}
 
template <typename Obj>
auto Substitute(Obj&& obj, const ffi::Map<Var, Expr>& vmap) {
  auto func = [&vmap](const Var& var) -> ffi::Optional<Expr> { return vmap.Get(var); };
  return Substitute(std::forward<Obj>(obj), func);
}
 
template <typename Obj, typename Replacement>
auto Substitute(Obj&& obj, const ffi::Map<Var, Replacement>& vmap) {
  auto func = [&vmap](const Var& var) -> ffi::Optional<Expr> {
    if (auto replacement = vmap.Get(var)) return Expr(replacement.value());
    return std::nullopt;
  };
  return Substitute(std::forward<Obj>(obj), func);
}
 
template <typename Obj, typename Replacement>
auto Substitute(Obj&& obj, const std::unordered_map<const VarNode*, Replacement>& vmap) {
  auto func = [&vmap](const Var& var) -> ffi::Optional<Expr> {
    if (auto it = vmap.find(var.get()); it != vmap.end()) {
      return Expr(it->second);
    }
    return std::nullopt;
  };
  return Substitute(std::forward<Obj>(obj), func);
}
 
template <typename Obj, typename Replacement, typename Hasher, typename EqualityChecker>
auto Substitute(Obj&& obj,
                const std::unordered_map<Var, Replacement, Hasher, EqualityChecker>& vmap) {
  auto func = [&vmap](const Var& var) -> ffi::Optional<Expr> {
    if (auto it = vmap.find(var); it != vmap.end()) {
      return Expr(it->second);
    }
    return std::nullopt;
  };
  return Substitute(std::forward<Obj>(obj), func);
}
 
template <typename Obj, typename Replacement>
auto Substitute(Obj&& obj, const std::unordered_map<IterVar, Replacement>& iter_vmap) {
  std::unordered_map<const VarNode*, Expr> vmap;
  for (const auto& [iter_var, expr] : iter_vmap) {
    vmap[iter_var->var.get()] = Expr(expr);
  }
 
  auto func = [&vmap](const Var& var) -> ffi::Optional<Expr> {
    if (auto it = vmap.find(var.get()); it != vmap.end()) {
      return it->second;
    } else {
      return std::nullopt;
    }
  };
  return Substitute(std::forward<Obj>(obj), func);
}
 
TVM_DLL Stmt SubstituteWithDataTypeLegalization(
    Stmt stmt, std::function<ffi::Optional<PrimExpr>(const Var&)> vmap);
 
TVM_DLL PrimExpr SubstituteWithDataTypeLegalization(
    PrimExpr expr, std::function<ffi::Optional<PrimExpr>(const Var&)> vmap);
 
TVM_DLL void PreOrderVisit(const ffi::ObjectRef& stmt_or_expr,
                           const std::function<bool(const ffi::ObjectRef&)>& fvisit);
 
template <typename Node, typename = std::enable_if_t<std::is_base_of_v<StmtNode, Node>>>
bool ContainsNode(const Stmt& stmt) {
  struct Visitor : StmtVisitor {
    // Early bail-out, if we already found the node.
    void VisitStmt(const Stmt& stmt) final {
      if (contains_node) {
        return;
      }
      StmtVisitor::VisitStmt(stmt);
    }
 
    void VisitStmt_(const Node* block) override { contains_node = true; }
 
    bool contains_node{false};
  };
 
  Visitor visitor;
  visitor(stmt);
  return visitor.contains_node;
}
 
}  // namespace tirx
}  // namespace tvm
 
#endif  // TVM_TIR_STMT_FUNCTOR_H_