memory_manager.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#ifndef TVM_RUNTIME_MEMORY_MEMORY_MANAGER_H_
#define TVM_RUNTIME_MEMORY_MEMORY_MANAGER_H_
 
#include <tvm/runtime/c_runtime_api.h>
#include <tvm/runtime/ndarray.h>
#include <tvm/runtime/object.h>
 
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
 
namespace tvm {
namespace runtime {
namespace memory {
 
enum AllocatorType {
  kNaive = 1,
  kPooled,
};
 
struct Buffer {
  void* data{nullptr};
  size_t size{0};
  Device device;
  AllocatorType alloc_type;
};
 
class Allocator {
 public:
  explicit Allocator(AllocatorType type) : type_(type) {}
  virtual ~Allocator() = default;
  NDArray Empty(ShapeTuple shape, DLDataType dtype, Device dev,
                Optional<String> mem_scope = NullOpt);
  inline AllocatorType type() const { return type_; }
  virtual Buffer Alloc(size_t nbytes, size_t alignment, DLDataType type_hint) = 0;
  virtual Buffer Alloc(ShapeTuple shape, DLDataType type_hint,
                       const std::string& mem_scope = "") = 0;
  virtual void Free(const Buffer& buffer) = 0;
  virtual void Clear();
  virtual size_t UsedMemory() const = 0;
 
 protected:
  virtual Buffer Alloc(Device dev, ShapeTuple shape, DLDataType type_hint,
                       const std::string& mem_scope);
 
 private:
  AllocatorType type_;
};
 
class MemoryManager {
 public:
  static MemoryManager* Global();
  static Allocator* GetOrCreateAllocator(Device dev, AllocatorType type);
  static Allocator* GetAllocator(Device dev, AllocatorType type);
  static void Clear();
 
 private:
  MemoryManager() {}
 
 protected:
  std::mutex mu_;
  std::unordered_map<Device, std::unordered_map<AllocatorType, std::unique_ptr<Allocator>>>
      allocators_;
};
 
class StorageObj : public Object {
 public:
  Buffer buffer;
 
  NDArray AllocNDArray(size_t offset, ShapeTuple shape, DLDataType dtype);
 
  static void Deleter(Object* ptr);
 
  ~StorageObj() {
    auto alloc = MemoryManager::Global()->GetAllocator(buffer.device, buffer.alloc_type);
    alloc->Free(buffer);
  }
 
  static constexpr const uint32_t _type_index = TypeIndex::kDynamic;
  static constexpr const char* _type_key = "vm.Storage";
  TVM_DECLARE_FINAL_OBJECT_INFO(StorageObj, Object);
};
 
class Storage : public ObjectRef {
 public:
  explicit Storage(Buffer buffer);
 
  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(Storage, ObjectRef, StorageObj);
};
 
}  // namespace memory
}  // namespace runtime
}  // namespace tvm
 
#endif  // TVM_RUNTIME_MEMORY_MEMORY_MANAGER_H_