ndarray.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#ifndef TVM_RUNTIME_NDARRAY_H_
#define TVM_RUNTIME_NDARRAY_H_
 
#include <tvm/runtime/c_runtime_api.h>
#include <tvm/runtime/container/optional.h>
#include <tvm/runtime/container/shape_tuple.h>
#include <tvm/runtime/container/string.h>
#include <tvm/runtime/data_type.h>
#include <tvm/runtime/object.h>
#include <tvm/runtime/serializer.h>
 
#include <atomic>
#include <functional>
#include <utility>
#include <vector>
 
namespace tvm {
 
// alias DLDevice
using Device = DLDevice;
 
namespace runtime {
 
class NDArray : public ObjectRef {
 public:
  class ContainerBase;
  class Container;
  using ContainerType = Container;
  NDArray() {}
  explicit NDArray(ObjectPtr<Object> data) : ObjectRef(data) {}
 
  inline void reset();
  inline int use_count() const;
  inline const DLTensor* operator->() const;
  inline bool IsContiguous() const;
  inline void CopyFrom(const DLTensor* other);
  inline void CopyFrom(const NDArray& other);
  TVM_DLL void CopyFromBytes(const void* data, size_t nbytes);
  inline void CopyTo(DLTensor* other) const;
  inline void CopyTo(const NDArray& other) const;
  TVM_DLL void CopyToBytes(void* data, size_t nbytes) const;
  TVM_DLL NDArray CopyTo(const Device& dev, Optional<String> mem_scope = NullOpt) const;
  inline bool Load(dmlc::Stream* stream);
  inline void Save(dmlc::Stream* stream) const;
 
  TVM_DLL NDArray CreateView(ShapeTuple shape, DLDataType dtype, uint64_t relative_byte_offset = 0);
 
  TVM_DLL DLManagedTensor* ToDLPack() const;
  TVM_DLL static NDArray Empty(ShapeTuple shape, DLDataType dtype, Device dev,
                               Optional<String> mem_scope = NullOpt);
  TVM_DLL static NDArray FromExternalDLTensor(const DLTensor& dl_tensor);
  TVM_DLL static NDArray NewFromDLTensor(DLTensor* dl_tensor, const Device& dev);
  TVM_DLL static NDArray FromDLPack(DLManagedTensor* tensor);
  TVM_DLL static void CopyFromTo(const DLTensor* from, DLTensor* to,
                                 TVMStreamHandle stream = nullptr);
 
  TVM_DLL ShapeTuple Shape() const;
  TVM_DLL runtime::DataType DataType() const;
  TVM_DLL static bool AbilityOfZeroCopyForDLTensor(DLTensor* tensor, const Device& dev);
  // internal namespace
  struct Internal;
 
 private:
  TVM_DLL static bool IsAligned(const DLTensor& tensor);
 
 protected:
  friend class TVMPODValue_;
  template <typename Derived>
  friend class TVMPODValue_CRTP_;
  friend class TVMRetValue;
  friend class TVMArgsSetter;
  inline Container* get_mutable() const;
  // Helper functions for FFI handling.
  inline static ObjectPtr<Object> FFIDataFromHandle(TVMArrayHandle handle);
  inline static void FFIDecRef(TVMArrayHandle handle);
  inline static TVMArrayHandle FFIGetHandle(const ObjectRef& nd);
};
 
inline bool SaveDLTensor(dmlc::Stream* strm, const DLTensor* tensor);
 
class NDArray::ContainerBase {
 public:
  DLTensor dl_tensor;
 
  void* manager_ctx{nullptr};
 
 protected:
  ShapeTuple shape_;
};
 
class NDArray::Container : public Object, public NDArray::ContainerBase {
 public:
  Container() {
    // Initialize the type index.
    type_index_ = Container::RuntimeTypeIndex();
    dl_tensor.data = nullptr;
    dl_tensor.ndim = 0;
    dl_tensor.shape = nullptr;
    dl_tensor.strides = nullptr;
    dl_tensor.byte_offset = 0;
  }
 
  Container(void* data, ShapeTuple shape, DLDataType dtype, Device dev) {
    // Initialize the type index.
    type_index_ = Container::RuntimeTypeIndex();
    dl_tensor.data = data;
    shape_ = std::move(shape);
    dl_tensor.ndim = static_cast<int>(shape_.size());
    dl_tensor.shape = const_cast<ShapeTuple::index_type*>(shape_.data());
    dl_tensor.dtype = dtype;
    dl_tensor.strides = nullptr;
    dl_tensor.byte_offset = 0;
    dl_tensor.device = dev;
  }
  void SetDeleter(FDeleter deleter) { deleter_ = deleter; }
 
  // Expose DecRef and IncRef as public function
  // NOTE: they are only for developer purposes only.
  using Object::DecRef;
  using Object::IncRef;
 
  // Information for object protocol.
  static constexpr const uint32_t _type_index = TypeIndex::kRuntimeNDArray;
  static constexpr const uint32_t _type_child_slots = 0;
  static constexpr const uint32_t _type_child_slots_can_overflow = true;
  static constexpr const char* _type_key = "runtime.NDArray";
  TVM_DECLARE_BASE_OBJECT_INFO(NDArray::Container, Object);
 
 protected:
  friend class RPCWrappedFunc;
  friend class NDArray;
};
 
// implementations of inline functions
inline size_t GetDataSize(const DLTensor& arr) {
  size_t size = 1;
  for (tvm_index_t i = 0; i < arr.ndim; ++i) {
    size *= static_cast<size_t>(arr.shape[i]);
  }
  size *= (arr.dtype.bits * arr.dtype.lanes + 7) / 8;
  return size;
}
 
static inline bool IsContiguous(const DLTensor& arr) {
  if (arr.strides == nullptr) return true;
  int64_t expected_stride = 1;
  for (int32_t i = arr.ndim; i != 0; --i) {
    int32_t k = i - 1;
    if (arr.shape[k] == 1) {
      // Skip stride check if shape[k] is 1, where the dimension is contiguous
      // regardless of the value of stride.
      //
      // For example, PyTorch will normalize stride to 1 if shape is 1 when exporting
      // to DLPack.
      // More context: https://github.com/pytorch/pytorch/pull/83158
      continue;
    }
    if (arr.strides[k] != expected_stride) return false;
    expected_stride *= arr.shape[k];
  }
  return true;
}
 
inline bool NDArray::IsContiguous() const {
  return ::tvm::runtime::IsContiguous(get_mutable()->dl_tensor);
}
 
inline void NDArray::CopyFrom(const DLTensor* other) {
  ICHECK(data_ != nullptr);
  CopyFromTo(other, &(get_mutable()->dl_tensor));
}
 
inline void NDArray::CopyFrom(const NDArray& other) {
  ICHECK(data_ != nullptr);
  ICHECK(other.data_ != nullptr);
  CopyFromTo(&(other.get_mutable()->dl_tensor), &(get_mutable()->dl_tensor));
}
 
inline void NDArray::CopyTo(DLTensor* other) const {
  ICHECK(data_ != nullptr);
  CopyFromTo(&(get_mutable()->dl_tensor), other);
}
 
inline void NDArray::CopyTo(const NDArray& other) const {
  ICHECK(data_ != nullptr);
  ICHECK(other.data_ != nullptr);
  CopyFromTo(&(get_mutable()->dl_tensor), &(other.get_mutable()->dl_tensor));
}
 
inline int NDArray::use_count() const { return data_.use_count(); }
 
inline const DLTensor* NDArray::operator->() const { return &(get_mutable()->dl_tensor); }
 
inline NDArray::Container* NDArray::get_mutable() const {
  return static_cast<NDArray::Container*>(data_.get());
}
 
inline ObjectPtr<Object> NDArray::FFIDataFromHandle(TVMArrayHandle handle) {
  return GetObjectPtr<Object>(
      static_cast<NDArray::Container*>(reinterpret_cast<NDArray::ContainerBase*>(handle)));
}
 
inline TVMArrayHandle NDArray::FFIGetHandle(const ObjectRef& nd) {
  // NOTE: it is necessary to cast to container then to base
  //       so that the FFI handle uses the ContainerBase address.
  auto ptr = reinterpret_cast<TVMArrayHandle>(static_cast<NDArray::ContainerBase*>(
      static_cast<NDArray::Container*>(const_cast<Object*>(nd.get()))));
  return ptr;
}
 
inline void NDArray::FFIDecRef(TVMArrayHandle handle) {
  static_cast<NDArray::Container*>(reinterpret_cast<NDArray::ContainerBase*>(handle))->DecRef();
}
 
inline Object* TVMArrayHandleToObjectHandle(TVMArrayHandle handle) {
  return static_cast<NDArray::Container*>(reinterpret_cast<NDArray::ContainerBase*>(handle));
}
 
constexpr uint64_t kTVMNDArrayMagic = 0xDD5E40F096B4A13F;
 
inline bool SaveDLTensor(dmlc::Stream* strm, const DLTensor* tensor) {
  uint64_t header = kTVMNDArrayMagic, reserved = 0;
  strm->Write(header);
  strm->Write(reserved);
  // Always save data as CPU context
  //
  // Parameters that get serialized should be in CPU by default.
  // So even the array's context is GPU, it will be stored as CPU array.
  // This is used to prevent case when another user loads the parameters
  // back on machine that do not have GPU or related context.
  //
  // We can always do array.CopyTo(target_dev) to get a corresponding
  // array in the target context.
  Device cpu_dev;
  cpu_dev.device_type = kDLCPU;
  cpu_dev.device_id = 0;
  strm->Write(cpu_dev);
  strm->Write(tensor->ndim);
  strm->Write(tensor->dtype);
  int ndim = tensor->ndim;
  strm->WriteArray(tensor->shape, ndim);
  int type_bytes = (tensor->dtype.bits + 7) / 8;
  int64_t num_elems = 1;
  for (int i = 0; i < ndim; ++i) {
    num_elems *= tensor->shape[i];
  }
  int64_t data_byte_size = type_bytes * num_elems;
  strm->Write(data_byte_size);
 
  if (DMLC_IO_NO_ENDIAN_SWAP && tensor->device.device_type == kDLCPU &&
      tensor->strides == nullptr && tensor->byte_offset == 0) {
    // quick path
    strm->Write(tensor->data, data_byte_size);
  } else {
    std::vector<uint8_t> bytes(data_byte_size);
    ICHECK_EQ(
        TVMArrayCopyToBytes(const_cast<DLTensor*>(tensor), dmlc::BeginPtr(bytes), data_byte_size),
        0)
        << TVMGetLastError();
    if (!DMLC_IO_NO_ENDIAN_SWAP) {
      dmlc::ByteSwap(dmlc::BeginPtr(bytes), type_bytes, num_elems);
    }
    strm->Write(dmlc::BeginPtr(bytes), data_byte_size);
  }
  return true;
}
 
inline void NDArray::Save(dmlc::Stream* strm) const { SaveDLTensor(strm, operator->()); }
 
inline bool NDArray::Load(dmlc::Stream* strm) {
  uint64_t header, reserved;
  ICHECK(strm->Read(&header)) << "Invalid DLTensor file format";
  ICHECK(strm->Read(&reserved)) << "Invalid DLTensor file format";
  ICHECK(header == kTVMNDArrayMagic) << "Invalid DLTensor file format";
  Device dev;
  int ndim;
  DLDataType dtype;
  ICHECK(strm->Read(&dev)) << "Invalid DLTensor file format";
  ICHECK(strm->Read(&ndim)) << "Invalid DLTensor file format";
  ICHECK(strm->Read(&dtype)) << "Invalid DLTensor file format";
  ICHECK_EQ(dev.device_type, kDLCPU) << "Invalid DLTensor device: can only save as CPU tensor";
  std::vector<int64_t> shape(ndim);
  if (ndim != 0) {
    ICHECK(strm->ReadArray(&shape[0], ndim)) << "Invalid DLTensor file format";
  }
  NDArray ret = NDArray::Empty(ShapeTuple(shape), dtype, dev);
  int64_t num_elems = 1;
  int elem_bytes = (ret->dtype.bits + 7) / 8;
  for (int i = 0; i < ret->ndim; ++i) {
    num_elems *= ret->shape[i];
  }
  int64_t data_byte_size;
  ICHECK(strm->Read(&data_byte_size)) << "Invalid DLTensor file format";
  ICHECK(data_byte_size == num_elems * elem_bytes) << "Invalid DLTensor file format";
  auto read_ret = strm->Read(ret->data, data_byte_size);
  // Only check non-empty data
  if (ndim > 0 && shape[0] != 0) {
    ICHECK(read_ret) << "Invalid DLTensor file format";
  }
  if (!DMLC_IO_NO_ENDIAN_SWAP) {
    dmlc::ByteSwap(ret->data, elem_bytes, num_elems);
  }
  *this = ret;
  return true;
}
 
inline Device GetPreferredHostDevice(Device device) {
  if (device.device_type == DLDeviceType::kDLCUDA) {
    return Device{DLDeviceType::kDLCUDAHost, 0};
  } else if (device.device_type == DLDeviceType::kDLROCM) {
    return Device{DLDeviceType::kDLROCMHost, 0};
  } else {
    // Fallback to CPU.
    return Device{DLDeviceType::kDLCPU, 0};
  }
}
 
}  // namespace runtime
}  // namespace tvm
 
namespace std {
template <>
struct hash<tvm::Device> {
  std::size_t operator()(const tvm::Device& dev) const {
    return ((dev.device_id << 8) | dev.device_type);
  }
};
 
template <>
struct equal_to<tvm::Device> {
  bool operator()(const tvm::Device& lhs, const tvm::Device& rhs) const {
    return (lhs.device_type == rhs.device_type && lhs.device_id == rhs.device_id);
  }
};
}  // namespace std
 
#endif  // TVM_RUNTIME_NDARRAY_H_