feat: rework physical allocator
This commit is contained in:
parent
e2ea22ad53
commit
bfa71d5836
4 changed files with 364 additions and 44 deletions
|
|
@ -34,7 +34,7 @@ static void step(const char* name, T&& fn) {
|
|||
print("\n");
|
||||
}
|
||||
|
||||
optional<paddr_t> findInitialMemoryPage(const multiboot::tag_mmap* multiboot_info) {
|
||||
optional<paddr_t> findInitialMemoryPages(const multiboot::tag_mmap* multiboot_info, size_t count) {
|
||||
const auto* base = reinterpret_cast<const uint8_t*>(multiboot_info->entries);
|
||||
const auto* end = reinterpret_cast<const uint8_t*>(multiboot_info) + multiboot_info->size;
|
||||
for (const auto* ptr = base; ptr < end; ptr += multiboot_info->entry_size) {
|
||||
|
|
@ -42,13 +42,13 @@ optional<paddr_t> findInitialMemoryPage(const multiboot::tag_mmap* multiboot_inf
|
|||
if(entry->type != multiboot::mem_type::available) {
|
||||
continue;
|
||||
}
|
||||
auto start_ptr = (entry->addr + page_size - 1) & (~page_size);
|
||||
auto end_ptr = (entry->addr + entry->len) & (~page_size);
|
||||
auto start_ptr = (entry->addr + page_size - 1) & ~(page_size - 1);
|
||||
auto end_ptr = (entry->addr + entry->len) & ~(page_size - 1);
|
||||
auto size = end_ptr - start_ptr;
|
||||
if(size < page_size) {
|
||||
if(size < page_size * count) {
|
||||
continue;
|
||||
}
|
||||
for(auto ptr = start_ptr; ptr < end_ptr; ptr += page_size) {
|
||||
for(auto ptr = start_ptr; ptr < (end_ptr - page_size * (count - 1)); ptr += page_size) {
|
||||
if(!is_in_reserved_section(paddr_t{ptr}, page_size)) {
|
||||
return paddr_t{ptr};
|
||||
}
|
||||
|
|
@ -59,16 +59,16 @@ optional<paddr_t> findInitialMemoryPage(const multiboot::tag_mmap* multiboot_inf
|
|||
|
||||
void PhysicalAllocator::init(const multiboot::tag_mmap* multiboot_info) {
|
||||
PhysicalAllocator alloc;
|
||||
auto page_opt = findInitialMemoryPage(multiboot_info);
|
||||
|
||||
auto page_opt = findInitialMemoryPages(multiboot_info, 1);
|
||||
if(!page_opt.has_value()) {
|
||||
panic("System could not find enough valid ram!");
|
||||
}
|
||||
auto page = page_opt.value();
|
||||
memset(page.access<uint8_t*>(), 0, page_size);
|
||||
alloc.storage = page.access<PhysicalAllocator::Page>();
|
||||
|
||||
alloc.storage->entries[0].page_count=68719476736;
|
||||
alloc.storage->entries[0].owner=pid_reserved;
|
||||
alloc.storage = page.access<PhysicalAllocator::page_t>();
|
||||
|
||||
alloc.storage->count[0] = (1ULL << 52);
|
||||
|
||||
step("multiboot", [&]() {
|
||||
const auto* base = reinterpret_cast<const uint8_t*>(multiboot_info->entries);
|
||||
|
|
@ -78,25 +78,28 @@ void PhysicalAllocator::init(const multiboot::tag_mmap* multiboot_info) {
|
|||
if(entry->type!=multiboot::mem_type::available) {
|
||||
continue;
|
||||
}
|
||||
auto start_ptr = (entry->addr + page_size - 1) & (~page_size);
|
||||
auto end_ptr = (entry->addr + entry->len) & (~page_size);
|
||||
auto size = end_ptr - start_ptr;
|
||||
alloc.storage->forceEntry({size/page_size, pid_free});
|
||||
auto start_ptr = (entry->addr + page_size - 1) & ~(page_size - 1);
|
||||
auto end_ptr = (entry->addr + entry->len) & ~(page_size - 1);
|
||||
if(end_ptr <= start_ptr) {
|
||||
continue;
|
||||
}
|
||||
alloc.force({paddr_t{start_ptr}, (end_ptr - start_ptr) / page_size}, true);
|
||||
}
|
||||
});
|
||||
|
||||
step("reserved_sections", [&]() {
|
||||
for_each_reserved_section([&](const KernelSection& section){
|
||||
if(section.vma_start == section.vma_end) {
|
||||
return;
|
||||
}
|
||||
auto start_ptr = (section.phys_start().address) & (~page_size);
|
||||
auto end_ptr = (section.phys_end().address + page_size - 1) & (~page_size);
|
||||
auto size = end_ptr - start_ptr;
|
||||
alloc.storage->forceEntry({size/page_size, pid_kernel});
|
||||
auto start_ptr = section.phys_start().address & ~(page_size - 1);
|
||||
auto end_ptr = (section.phys_end().address + page_size - 1) & ~(page_size - 1);
|
||||
alloc.force({paddr_t{start_ptr}, (end_ptr - start_ptr) / page_size}, false);
|
||||
});
|
||||
});
|
||||
alloc.storage->cleanup();
|
||||
alloc.storage->compact();
|
||||
|
||||
alloc.force({page, 1}, false);
|
||||
|
||||
PhysicalAllocator::getInstance() = alloc;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,14 +3,334 @@
|
|||
|
||||
static PhysicalAllocator allocator{};
|
||||
|
||||
struct PhysicalAllocatorImpl {
|
||||
using page_t = PhysicalAllocator::page_t;
|
||||
using allocation_t = PhysicalAllocator::allocation_t;
|
||||
|
||||
static constexpr size_t entries_per_node = 510;
|
||||
static constexpr size_t set_range_margin = 4;
|
||||
|
||||
PhysicalAllocator& alloc;
|
||||
|
||||
struct Cursor {
|
||||
page_t* node;
|
||||
size_t slot;
|
||||
uint64_t& value() { return node->count[slot]; }
|
||||
void advance() {
|
||||
++slot;
|
||||
if (slot == entries_per_node) {
|
||||
node = node->next;
|
||||
slot = 0;
|
||||
}
|
||||
}
|
||||
void retreat() {
|
||||
if (slot == 0) {
|
||||
node = node->prev;
|
||||
slot = entries_per_node - 1;
|
||||
} else {
|
||||
--slot;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct run_location_t {
|
||||
size_t index;
|
||||
uint64_t address;
|
||||
uint64_t length;
|
||||
};
|
||||
struct run_t {
|
||||
bool is_free;
|
||||
uint64_t length;
|
||||
};
|
||||
struct run_list_t {
|
||||
run_t runs[6];
|
||||
size_t count;
|
||||
};
|
||||
|
||||
Cursor cursor_at(size_t index) {
|
||||
Cursor cursor{alloc.storage, 0};
|
||||
for (size_t hop = index / entries_per_node; hop > 0; --hop) {
|
||||
cursor.node = cursor.node->next;
|
||||
}
|
||||
cursor.slot = index % entries_per_node;
|
||||
return cursor;
|
||||
}
|
||||
|
||||
uint64_t& entry(size_t i) {
|
||||
return cursor_at(i).value();
|
||||
}
|
||||
|
||||
size_t node_count() {
|
||||
size_t nodes = 0;
|
||||
for (page_t* node = alloc.storage; node; node = node->next) {
|
||||
++nodes;
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
size_t capacity() {
|
||||
return node_count() * entries_per_node;
|
||||
}
|
||||
|
||||
size_t logical_length() {
|
||||
Cursor cursor = cursor_at(0);
|
||||
size_t i = 0;
|
||||
bool prev_zero = false;
|
||||
while (true) {
|
||||
uint64_t length = cursor.value();
|
||||
if (length == 0 && prev_zero) {
|
||||
return i - 1;
|
||||
}
|
||||
prev_zero = (length == 0);
|
||||
cursor.advance();
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
size_t free_slots() {
|
||||
return capacity() - (logical_length() + 2);
|
||||
}
|
||||
|
||||
optional<paddr_t> find_free_page() {
|
||||
Cursor cursor = cursor_at(0);
|
||||
uint64_t page = 0;
|
||||
size_t i = 0;
|
||||
bool prev_zero = false;
|
||||
while (true) {
|
||||
uint64_t length = cursor.value();
|
||||
if (length == 0 && prev_zero) {
|
||||
return {};
|
||||
}
|
||||
if ((i & 1) && length > 0) {
|
||||
return paddr_t{page * page_size};
|
||||
}
|
||||
page += length;
|
||||
prev_zero = (length == 0);
|
||||
cursor.advance();
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
bool ensure_free_slots(size_t needed) {
|
||||
while (free_slots() < needed) {
|
||||
auto free = find_free_page();
|
||||
if (!free.has_value()) {
|
||||
return false;
|
||||
}
|
||||
auto page = free.value();
|
||||
page_t* node = page.access<page_t>();
|
||||
memset(node, 0, page_size);
|
||||
page_t* last = alloc.storage;
|
||||
while (last->next) {
|
||||
last = last->next;
|
||||
}
|
||||
last->next = node;
|
||||
node->prev = last;
|
||||
set_range(page.address / page_size, 1, false);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void reclaim_nodes() {
|
||||
while (node_count() > 1) {
|
||||
size_t capacity_without_last = (node_count() - 1) * entries_per_node;
|
||||
if (capacity_without_last < logical_length() + 2 + set_range_margin) {
|
||||
break;
|
||||
}
|
||||
page_t* prev = alloc.storage;
|
||||
while (prev->next->next) {
|
||||
prev = prev->next;
|
||||
}
|
||||
page_t* last = prev->next;
|
||||
prev->next = nullptr;
|
||||
uint64_t page = (reinterpret_cast<uint64_t>(last) - high_base) / page_size;
|
||||
set_range(page, 1, true);
|
||||
}
|
||||
}
|
||||
|
||||
optional<run_location_t> locate_run(uint64_t page) {
|
||||
Cursor cursor = cursor_at(0);
|
||||
uint64_t address = 0;
|
||||
size_t i = 0;
|
||||
bool prev_zero = false;
|
||||
while (true) {
|
||||
uint64_t length = cursor.value();
|
||||
if (length == 0 && prev_zero) {
|
||||
return {};
|
||||
}
|
||||
if (page >= address && page < address + length) {
|
||||
return run_location_t{i, address, length};
|
||||
}
|
||||
address += length;
|
||||
prev_zero = (length == 0);
|
||||
cursor.advance();
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
run_list_t collect_runs(uint64_t start_page, uint64_t count, bool is_free,
|
||||
const run_location_t& start, const run_location_t& end, size_t total_runs) {
|
||||
bool has_left_neighbor = start.index > 0;
|
||||
bool has_right_neighbor = end.index + 1 < total_runs;
|
||||
uint64_t left_remainder = start_page - start.address;
|
||||
uint64_t right_remainder = (end.address + end.length) - (start_page + count);
|
||||
|
||||
run_list_t list{};
|
||||
if (has_left_neighbor) {
|
||||
list.runs[list.count++] = { ((start.index - 1) & 1) != 0, entry(start.index - 1) };
|
||||
}
|
||||
list.runs[list.count++] = { (start.index & 1) != 0, left_remainder };
|
||||
list.runs[list.count++] = { is_free, count };
|
||||
list.runs[list.count++] = { (end.index & 1) != 0, right_remainder };
|
||||
if (has_right_neighbor) {
|
||||
list.runs[list.count++] = { ((end.index + 1) & 1) != 0, entry(end.index + 1) };
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
static run_list_t merge_runs(const run_list_t& in) {
|
||||
run_list_t out{};
|
||||
for (size_t i = 0; i < in.count; ++i) {
|
||||
if (in.runs[i].length == 0) {
|
||||
continue;
|
||||
}
|
||||
if (out.count > 0 && out.runs[out.count - 1].is_free == in.runs[i].is_free) {
|
||||
out.runs[out.count - 1].length += in.runs[i].length;
|
||||
} else {
|
||||
out.runs[out.count++] = in.runs[i];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static run_list_t restore_leading_parity(run_list_t list, bool first_is_free) {
|
||||
if (list.count > 0 && list.runs[0].is_free == first_is_free) {
|
||||
return list;
|
||||
}
|
||||
for (size_t i = list.count; i > 0; --i) {
|
||||
list.runs[i] = list.runs[i - 1];
|
||||
}
|
||||
list.runs[0] = { first_is_free, 0 };
|
||||
++list.count;
|
||||
return list;
|
||||
}
|
||||
|
||||
void splice_runs(size_t first_index, size_t last_index, const run_list_t& list, size_t total_runs) {
|
||||
size_t old_count = last_index - first_index + 1;
|
||||
size_t terminator_end = total_runs + 2;
|
||||
if (list.count > old_count) {
|
||||
size_t delta = list.count - old_count;
|
||||
Cursor source = cursor_at(terminator_end - 1);
|
||||
Cursor dest = cursor_at(terminator_end - 1 + delta);
|
||||
for (size_t i = terminator_end; i-- > last_index + 1; ) {
|
||||
dest.value() = source.value();
|
||||
source.retreat();
|
||||
dest.retreat();
|
||||
}
|
||||
} else if (list.count < old_count) {
|
||||
size_t delta = old_count - list.count;
|
||||
Cursor source = cursor_at(last_index + 1);
|
||||
Cursor dest = cursor_at(last_index + 1 - delta);
|
||||
for (size_t i = last_index + 1; i < terminator_end; ++i) {
|
||||
dest.value() = source.value();
|
||||
source.advance();
|
||||
dest.advance();
|
||||
}
|
||||
Cursor tail = cursor_at(terminator_end - delta);
|
||||
for (size_t i = terminator_end - delta; i < terminator_end; ++i) {
|
||||
tail.value() = 0;
|
||||
tail.advance();
|
||||
}
|
||||
}
|
||||
Cursor write = cursor_at(first_index);
|
||||
for (size_t i = 0; i < list.count; ++i) {
|
||||
write.value() = list.runs[i].length;
|
||||
write.advance();
|
||||
}
|
||||
}
|
||||
|
||||
void set_range(uint64_t start_page, uint64_t count, bool is_free) {
|
||||
if (count == 0) {
|
||||
return;
|
||||
}
|
||||
auto start = locate_run(start_page);
|
||||
auto end = locate_run(start_page + count - 1);
|
||||
if (!start.has_value() || !end.has_value()) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t total_runs = logical_length();
|
||||
size_t first_index = start->index > 0 ? start->index - 1 : start->index;
|
||||
size_t last_index = end->index + 1 < total_runs ? end->index + 1 : end->index;
|
||||
|
||||
run_list_t collected = collect_runs(start_page, count, is_free, start.value(), end.value(), total_runs);
|
||||
run_list_t merged = merge_runs(collected);
|
||||
merged = restore_leading_parity(merged, (first_index & 1) != 0);
|
||||
|
||||
splice_runs(first_index, last_index, merged, total_runs);
|
||||
}
|
||||
|
||||
bool force(allocation_t ptr, bool is_free) {
|
||||
if (ptr.page_count == 0) {
|
||||
return true;
|
||||
}
|
||||
if (!ensure_free_slots(set_range_margin)) {
|
||||
return false;
|
||||
}
|
||||
set_range(ptr.ptr.address / page_size, ptr.page_count, is_free);
|
||||
reclaim_nodes();
|
||||
return true;
|
||||
}
|
||||
|
||||
optional<allocation_t> allocPages(size_t page_count, size_t alignment_pages) {
|
||||
if (page_count == 0) {
|
||||
return {};
|
||||
}
|
||||
if (alignment_pages == 0) {
|
||||
alignment_pages = 1;
|
||||
}
|
||||
if (!ensure_free_slots(set_range_margin)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
Cursor cursor = cursor_at(0);
|
||||
uint64_t run_start = 0;
|
||||
size_t i = 0;
|
||||
bool prev_zero = false;
|
||||
while (true) {
|
||||
uint64_t length = cursor.value();
|
||||
if (length == 0 && prev_zero) {
|
||||
break;
|
||||
}
|
||||
if ((i & 1) && length > 0) {
|
||||
uint64_t aligned_start = ((run_start + alignment_pages - 1) / alignment_pages) * alignment_pages;
|
||||
if (aligned_start + page_count <= run_start + length) {
|
||||
set_range(aligned_start, page_count, false);
|
||||
reclaim_nodes();
|
||||
return allocation_t{ paddr_t{aligned_start * page_size}, page_count };
|
||||
}
|
||||
}
|
||||
run_start += length;
|
||||
prev_zero = (length == 0);
|
||||
cursor.advance();
|
||||
++i;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
PhysicalAllocator& PhysicalAllocator::getInstance() {
|
||||
return allocator;
|
||||
}
|
||||
|
||||
PhysicalAllocator::allocation_t PhysicalAllocator::allocPages(size_t count, pid_t owner) {
|
||||
bool PhysicalAllocator::force(allocation_t ptr, bool free) {
|
||||
return PhysicalAllocatorImpl{*this}.force(ptr, free);
|
||||
}
|
||||
|
||||
optional<PhysicalAllocator::allocation_t> PhysicalAllocator::allocPages(size_t page_count, size_t alignment_pages) {
|
||||
return PhysicalAllocatorImpl{*this}.allocPages(page_count, alignment_pages);
|
||||
}
|
||||
|
||||
void PhysicalAllocator::free(allocation_t ptr) {
|
||||
|
||||
PhysicalAllocatorImpl{*this}.force(ptr, true);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue