diff --git a/.gitignore b/.gitignore index d373a4d..87c8c03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .claude .vscode compile_commands.json -CLAUDE.md \ No newline at end of file +CLAUDE.md +debug \ No newline at end of file diff --git a/kernel/include/memory/allocator.h b/kernel/include/memory/allocator.h index b75130e..a257ff8 100644 --- a/kernel/include/memory/allocator.h +++ b/kernel/include/memory/allocator.h @@ -1,37 +1,33 @@ #pragma once #include "memory/pointer.h" -#include "process/process.h" #include "util/optional.h" #include "init/multiboot2.h" constexpr const size_t page_size = 4096; struct PhysicalAllocator { -public: struct allocation_t { paddr_t ptr; size_t page_count; }; - struct entry_t { - uint64_t page_count; - pid_t owner; - }; + static void init(const multiboot::tag_mmap* multiboot_info); static PhysicalAllocator& getInstance(); - allocation_t allocPages(size_t page_count, pid_t owner); - void free(allocation_t ptr); - void force(allocation_t ptr, pid_t owner); - entry_t getOwner(paddr_t from); -private: - struct Page { - optional next; - entry_t entries[255]; - void forceEntry(entry_t entry); - void cleanup(); - void compact(); - }; - static_assert(sizeof(Page) == page_size); - Page* storage; -}; + optional allocPages(size_t page_count, size_t alignment_pages = 1); + void free(allocation_t ptr); + bool force(allocation_t ptr, bool free); + +private: + struct page_t { + page_t* next; + page_t* prev; + uint64_t count[510]; + }; + static_assert(sizeof(page_t) == page_size); + + page_t* storage; + + friend struct PhysicalAllocatorImpl; +}; diff --git a/kernel/src/init/init.cpp b/kernel/src/init/init.cpp index 7c12be7..02d08d6 100644 --- a/kernel/src/init/init.cpp +++ b/kernel/src/init/init.cpp @@ -34,7 +34,7 @@ static void step(const char* name, T&& fn) { print("\n"); } -optional findInitialMemoryPage(const multiboot::tag_mmap* multiboot_info) { +optional findInitialMemoryPages(const multiboot::tag_mmap* multiboot_info, size_t count) { const auto* base = reinterpret_cast(multiboot_info->entries); const auto* end = reinterpret_cast(multiboot_info) + multiboot_info->size; for (const auto* ptr = base; ptr < end; ptr += multiboot_info->entry_size) { @@ -42,13 +42,13 @@ optional findInitialMemoryPage(const multiboot::tag_mmap* multiboot_inf if(entry->type != multiboot::mem_type::available) { continue; } - auto start_ptr = (entry->addr + page_size - 1) & (~page_size); - auto end_ptr = (entry->addr + entry->len) & (~page_size); + auto start_ptr = (entry->addr + page_size - 1) & ~(page_size - 1); + auto end_ptr = (entry->addr + entry->len) & ~(page_size - 1); auto size = end_ptr - start_ptr; - if(size < page_size) { + if(size < page_size * count) { continue; } - for(auto ptr = start_ptr; ptr < end_ptr; ptr += page_size) { + for(auto ptr = start_ptr; ptr < (end_ptr - page_size * (count - 1)); ptr += page_size) { if(!is_in_reserved_section(paddr_t{ptr}, page_size)) { return paddr_t{ptr}; } @@ -59,16 +59,16 @@ optional findInitialMemoryPage(const multiboot::tag_mmap* multiboot_inf void PhysicalAllocator::init(const multiboot::tag_mmap* multiboot_info) { PhysicalAllocator alloc; - auto page_opt = findInitialMemoryPage(multiboot_info); + + auto page_opt = findInitialMemoryPages(multiboot_info, 1); if(!page_opt.has_value()) { panic("System could not find enough valid ram!"); } auto page = page_opt.value(); memset(page.access(), 0, page_size); - alloc.storage = page.access(); - - alloc.storage->entries[0].page_count=68719476736; - alloc.storage->entries[0].owner=pid_reserved; + alloc.storage = page.access(); + + alloc.storage->count[0] = (1ULL << 52); step("multiboot", [&]() { const auto* base = reinterpret_cast(multiboot_info->entries); @@ -78,25 +78,28 @@ void PhysicalAllocator::init(const multiboot::tag_mmap* multiboot_info) { if(entry->type!=multiboot::mem_type::available) { continue; } - auto start_ptr = (entry->addr + page_size - 1) & (~page_size); - auto end_ptr = (entry->addr + entry->len) & (~page_size); - auto size = end_ptr - start_ptr; - alloc.storage->forceEntry({size/page_size, pid_free}); + auto start_ptr = (entry->addr + page_size - 1) & ~(page_size - 1); + auto end_ptr = (entry->addr + entry->len) & ~(page_size - 1); + if(end_ptr <= start_ptr) { + continue; + } + alloc.force({paddr_t{start_ptr}, (end_ptr - start_ptr) / page_size}, true); } }); + step("reserved_sections", [&]() { for_each_reserved_section([&](const KernelSection& section){ if(section.vma_start == section.vma_end) { return; } - auto start_ptr = (section.phys_start().address) & (~page_size); - auto end_ptr = (section.phys_end().address + page_size - 1) & (~page_size); - auto size = end_ptr - start_ptr; - alloc.storage->forceEntry({size/page_size, pid_kernel}); + auto start_ptr = section.phys_start().address & ~(page_size - 1); + auto end_ptr = (section.phys_end().address + page_size - 1) & ~(page_size - 1); + alloc.force({paddr_t{start_ptr}, (end_ptr - start_ptr) / page_size}, false); }); }); - alloc.storage->cleanup(); - alloc.storage->compact(); + + alloc.force({page, 1}, false); + PhysicalAllocator::getInstance() = alloc; } diff --git a/kernel/src/memory/allocator.cpp b/kernel/src/memory/allocator.cpp index 1685a49..50cef9e 100644 --- a/kernel/src/memory/allocator.cpp +++ b/kernel/src/memory/allocator.cpp @@ -3,14 +3,334 @@ static PhysicalAllocator allocator{}; +struct PhysicalAllocatorImpl { + using page_t = PhysicalAllocator::page_t; + using allocation_t = PhysicalAllocator::allocation_t; + + static constexpr size_t entries_per_node = 510; + static constexpr size_t set_range_margin = 4; + + PhysicalAllocator& alloc; + + struct Cursor { + page_t* node; + size_t slot; + uint64_t& value() { return node->count[slot]; } + void advance() { + ++slot; + if (slot == entries_per_node) { + node = node->next; + slot = 0; + } + } + void retreat() { + if (slot == 0) { + node = node->prev; + slot = entries_per_node - 1; + } else { + --slot; + } + } + }; + + struct run_location_t { + size_t index; + uint64_t address; + uint64_t length; + }; + struct run_t { + bool is_free; + uint64_t length; + }; + struct run_list_t { + run_t runs[6]; + size_t count; + }; + + Cursor cursor_at(size_t index) { + Cursor cursor{alloc.storage, 0}; + for (size_t hop = index / entries_per_node; hop > 0; --hop) { + cursor.node = cursor.node->next; + } + cursor.slot = index % entries_per_node; + return cursor; + } + + uint64_t& entry(size_t i) { + return cursor_at(i).value(); + } + + size_t node_count() { + size_t nodes = 0; + for (page_t* node = alloc.storage; node; node = node->next) { + ++nodes; + } + return nodes; + } + + size_t capacity() { + return node_count() * entries_per_node; + } + + size_t logical_length() { + Cursor cursor = cursor_at(0); + size_t i = 0; + bool prev_zero = false; + while (true) { + uint64_t length = cursor.value(); + if (length == 0 && prev_zero) { + return i - 1; + } + prev_zero = (length == 0); + cursor.advance(); + ++i; + } + } + + size_t free_slots() { + return capacity() - (logical_length() + 2); + } + + optional find_free_page() { + Cursor cursor = cursor_at(0); + uint64_t page = 0; + size_t i = 0; + bool prev_zero = false; + while (true) { + uint64_t length = cursor.value(); + if (length == 0 && prev_zero) { + return {}; + } + if ((i & 1) && length > 0) { + return paddr_t{page * page_size}; + } + page += length; + prev_zero = (length == 0); + cursor.advance(); + ++i; + } + } + + bool ensure_free_slots(size_t needed) { + while (free_slots() < needed) { + auto free = find_free_page(); + if (!free.has_value()) { + return false; + } + auto page = free.value(); + page_t* node = page.access(); + memset(node, 0, page_size); + page_t* last = alloc.storage; + while (last->next) { + last = last->next; + } + last->next = node; + node->prev = last; + set_range(page.address / page_size, 1, false); + } + return true; + } + + void reclaim_nodes() { + while (node_count() > 1) { + size_t capacity_without_last = (node_count() - 1) * entries_per_node; + if (capacity_without_last < logical_length() + 2 + set_range_margin) { + break; + } + page_t* prev = alloc.storage; + while (prev->next->next) { + prev = prev->next; + } + page_t* last = prev->next; + prev->next = nullptr; + uint64_t page = (reinterpret_cast(last) - high_base) / page_size; + set_range(page, 1, true); + } + } + + optional locate_run(uint64_t page) { + Cursor cursor = cursor_at(0); + uint64_t address = 0; + size_t i = 0; + bool prev_zero = false; + while (true) { + uint64_t length = cursor.value(); + if (length == 0 && prev_zero) { + return {}; + } + if (page >= address && page < address + length) { + return run_location_t{i, address, length}; + } + address += length; + prev_zero = (length == 0); + cursor.advance(); + ++i; + } + } + + run_list_t collect_runs(uint64_t start_page, uint64_t count, bool is_free, + const run_location_t& start, const run_location_t& end, size_t total_runs) { + bool has_left_neighbor = start.index > 0; + bool has_right_neighbor = end.index + 1 < total_runs; + uint64_t left_remainder = start_page - start.address; + uint64_t right_remainder = (end.address + end.length) - (start_page + count); + + run_list_t list{}; + if (has_left_neighbor) { + list.runs[list.count++] = { ((start.index - 1) & 1) != 0, entry(start.index - 1) }; + } + list.runs[list.count++] = { (start.index & 1) != 0, left_remainder }; + list.runs[list.count++] = { is_free, count }; + list.runs[list.count++] = { (end.index & 1) != 0, right_remainder }; + if (has_right_neighbor) { + list.runs[list.count++] = { ((end.index + 1) & 1) != 0, entry(end.index + 1) }; + } + return list; + } + + static run_list_t merge_runs(const run_list_t& in) { + run_list_t out{}; + for (size_t i = 0; i < in.count; ++i) { + if (in.runs[i].length == 0) { + continue; + } + if (out.count > 0 && out.runs[out.count - 1].is_free == in.runs[i].is_free) { + out.runs[out.count - 1].length += in.runs[i].length; + } else { + out.runs[out.count++] = in.runs[i]; + } + } + return out; + } + + static run_list_t restore_leading_parity(run_list_t list, bool first_is_free) { + if (list.count > 0 && list.runs[0].is_free == first_is_free) { + return list; + } + for (size_t i = list.count; i > 0; --i) { + list.runs[i] = list.runs[i - 1]; + } + list.runs[0] = { first_is_free, 0 }; + ++list.count; + return list; + } + + void splice_runs(size_t first_index, size_t last_index, const run_list_t& list, size_t total_runs) { + size_t old_count = last_index - first_index + 1; + size_t terminator_end = total_runs + 2; + if (list.count > old_count) { + size_t delta = list.count - old_count; + Cursor source = cursor_at(terminator_end - 1); + Cursor dest = cursor_at(terminator_end - 1 + delta); + for (size_t i = terminator_end; i-- > last_index + 1; ) { + dest.value() = source.value(); + source.retreat(); + dest.retreat(); + } + } else if (list.count < old_count) { + size_t delta = old_count - list.count; + Cursor source = cursor_at(last_index + 1); + Cursor dest = cursor_at(last_index + 1 - delta); + for (size_t i = last_index + 1; i < terminator_end; ++i) { + dest.value() = source.value(); + source.advance(); + dest.advance(); + } + Cursor tail = cursor_at(terminator_end - delta); + for (size_t i = terminator_end - delta; i < terminator_end; ++i) { + tail.value() = 0; + tail.advance(); + } + } + Cursor write = cursor_at(first_index); + for (size_t i = 0; i < list.count; ++i) { + write.value() = list.runs[i].length; + write.advance(); + } + } + + void set_range(uint64_t start_page, uint64_t count, bool is_free) { + if (count == 0) { + return; + } + auto start = locate_run(start_page); + auto end = locate_run(start_page + count - 1); + if (!start.has_value() || !end.has_value()) { + return; + } + + size_t total_runs = logical_length(); + size_t first_index = start->index > 0 ? start->index - 1 : start->index; + size_t last_index = end->index + 1 < total_runs ? end->index + 1 : end->index; + + run_list_t collected = collect_runs(start_page, count, is_free, start.value(), end.value(), total_runs); + run_list_t merged = merge_runs(collected); + merged = restore_leading_parity(merged, (first_index & 1) != 0); + + splice_runs(first_index, last_index, merged, total_runs); + } + + bool force(allocation_t ptr, bool is_free) { + if (ptr.page_count == 0) { + return true; + } + if (!ensure_free_slots(set_range_margin)) { + return false; + } + set_range(ptr.ptr.address / page_size, ptr.page_count, is_free); + reclaim_nodes(); + return true; + } + + optional allocPages(size_t page_count, size_t alignment_pages) { + if (page_count == 0) { + return {}; + } + if (alignment_pages == 0) { + alignment_pages = 1; + } + if (!ensure_free_slots(set_range_margin)) { + return {}; + } + + Cursor cursor = cursor_at(0); + uint64_t run_start = 0; + size_t i = 0; + bool prev_zero = false; + while (true) { + uint64_t length = cursor.value(); + if (length == 0 && prev_zero) { + break; + } + if ((i & 1) && length > 0) { + uint64_t aligned_start = ((run_start + alignment_pages - 1) / alignment_pages) * alignment_pages; + if (aligned_start + page_count <= run_start + length) { + set_range(aligned_start, page_count, false); + reclaim_nodes(); + return allocation_t{ paddr_t{aligned_start * page_size}, page_count }; + } + } + run_start += length; + prev_zero = (length == 0); + cursor.advance(); + ++i; + } + return {}; + } +}; + PhysicalAllocator& PhysicalAllocator::getInstance() { return allocator; } -PhysicalAllocator::allocation_t PhysicalAllocator::allocPages(size_t count, pid_t owner) { +bool PhysicalAllocator::force(allocation_t ptr, bool free) { + return PhysicalAllocatorImpl{*this}.force(ptr, free); +} +optional PhysicalAllocator::allocPages(size_t page_count, size_t alignment_pages) { + return PhysicalAllocatorImpl{*this}.allocPages(page_count, alignment_pages); } void PhysicalAllocator::free(allocation_t ptr) { - + PhysicalAllocatorImpl{*this}.force(ptr, true); } \ No newline at end of file