memory_manager.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#ifndef TVM_RUNTIME_VM_MEMORY_MANAGER_H_
#define TVM_RUNTIME_VM_MEMORY_MANAGER_H_

#include <tvm/runtime/c_runtime_api.h>
#include <tvm/runtime/ndarray.h>
#include <tvm/runtime/object.h>

#include <functional>
#include <memory>
#include <mutex>
#include <unordered_map>
#include <vector>

namespace tvm {
namespace runtime {
namespace vm {

struct Buffer {
  void* data{nullptr};
  size_t size{0};
  Device device;
};

enum AllocatorType {
  kNaive = 1,
  kPooled,
};

class Allocator {
 public:
  explicit Allocator(AllocatorType type) : type_(type) {}
  virtual ~Allocator() = default;
  NDArray Empty(std::vector<int64_t> shape, DLDataType dtype, Device dev);
  inline AllocatorType type() const { return type_; }
  virtual Buffer Alloc(size_t nbytes, size_t alignment, DLDataType type_hint) = 0;
  virtual void Free(const Buffer& buffer) = 0;
  virtual size_t UsedMemory() const = 0;

 private:
  AllocatorType type_;
};

class MemoryManager {
 public:
  static MemoryManager* Global();
  static Allocator* GetOrCreateAllocator(Device dev, AllocatorType type);
  static Allocator* GetAllocator(Device dev);

 private:
  MemoryManager() {}

 private:
  std::mutex mu_;
  std::unordered_map<Device, std::unique_ptr<Allocator>> allocators_;
};

class StorageObj : public Object {
 public:
  Buffer buffer;

  NDArray AllocNDArray(size_t offset, std::vector<int64_t> shape, DLDataType dtype);

  static void Deleter(Object* ptr);

  ~StorageObj() {
    auto alloc = MemoryManager::Global()->GetAllocator(buffer.device);
    alloc->Free(buffer);
  }

  static constexpr const uint32_t _type_index = TypeIndex::kDynamic;
  static constexpr const char* _type_key = "vm.Storage";
  TVM_DECLARE_FINAL_OBJECT_INFO(StorageObj, Object);
};

class Storage : public ObjectRef {
 public:
  explicit Storage(Buffer buffer);

  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(Storage, ObjectRef, StorageObj);
};

}  // namespace vm
}  // namespace runtime
}  // namespace tvm

#endif  // TVM_RUNTIME_VM_MEMORY_MANAGER_H_