memory_manager.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#ifndef TVM_RUNTIME_MEMORY_MEMORY_MANAGER_H_
#define TVM_RUNTIME_MEMORY_MEMORY_MANAGER_H_
 
#include <tvm/runtime/base.h>
#include <tvm/runtime/tensor.h>
 
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
 
namespace tvm {
namespace runtime {
namespace memory {
 
enum AllocatorType {
  kNaive = 1,
  kPooled,
};
 
struct Buffer {
  void* data{nullptr};
  size_t size{0};
  Device device;
  AllocatorType alloc_type;
};
 
class TVM_RUNTIME_DLL Allocator {
 public:
  explicit Allocator(AllocatorType type) : type_(type) {}
  virtual ~Allocator() = default;
  Tensor Empty(ffi::Shape shape, DLDataType dtype, Device dev,
               ffi::Optional<ffi::String> mem_scope = std::nullopt);
  inline AllocatorType type() const { return type_; }
  virtual Buffer Alloc(Device dev, size_t nbytes, size_t alignment, DLDataType type_hint) = 0;
  virtual Buffer Alloc(Device dev, ffi::Shape shape, DLDataType type_hint,
                       const std::string& mem_scope = "");
 
  virtual void* CreateView(const Buffer& buffer, ffi::Shape shape, DLDataType type_hint,
                           const std::string& mem_scope = "global") {
    return buffer.data;
  }
 
  virtual void FreeView(Device dev, void* data) {}
 
  virtual void Free(const Buffer& buffer) = 0;
  virtual void Clear();
  virtual size_t UsedMemory() const = 0;
 
 protected:
  virtual bool AllowMemoryScope(const std::string& mem_scope) const;
 
 private:
  AllocatorType type_;
};
 
class MemoryManager {
 public:
  TVM_RUNTIME_DLL static MemoryManager* Global();
  TVM_RUNTIME_DLL static Allocator* GetOrCreateAllocator(Device dev, AllocatorType type);
  TVM_RUNTIME_DLL static Allocator* GetAllocator(Device dev, AllocatorType type);
  static void Clear();
 
 private:
  MemoryManager() {}
 
 protected:
  std::mutex mu_;
  std::unordered_map<Device, std::unordered_map<AllocatorType, std::unique_ptr<Allocator>>>
      allocators_;
};
 
class StorageObj : public ffi::Object {
 public:
  Buffer buffer;
  Allocator* allocator = nullptr;
 
  TVM_RUNTIME_DLL Tensor AllocTensor(int64_t offset, ffi::Shape shape, DLDataType dtype);
 
  TVM_RUNTIME_DLL Tensor AllocTensorScoped(int64_t offset, ffi::Shape shape, DLDataType dtype,
                                           ffi::String scope = "global");
 
  ~StorageObj() {
    if (allocator) {
      allocator->Free(buffer);
    }
  }
 
  static constexpr const bool _type_mutable = true;
  TVM_FFI_DECLARE_OBJECT_INFO_FINAL("vm.Storage", StorageObj, ffi::Object);
};
 
class Storage : public ffi::ObjectRef {
 public:
  TVM_RUNTIME_DLL explicit Storage(Buffer buffer, Allocator* allocator);
 
  TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(Storage, ffi::ObjectRef, StorageObj);
};
 
}  // namespace memory
 
using memory::Allocator;
using memory::AllocatorType;
using memory::MemoryManager;
using memory::StorageObj;
 
}  // namespace runtime
}  // namespace tvm
 
#endif  // TVM_RUNTIME_MEMORY_MEMORY_MANAGER_H_