feat: rework physical allocator

.gitignore

@@ -2,3 +2,4 @@
 .vscode
 compile_commands.json
 CLAUDE.md
+debug

kernel/include/memory/allocator.h

@@ -1,37 +1,33 @@
 #pragma once
 
 #include "memory/pointer.h"
-#include "process/process.h"
 #include "util/optional.h"
 #include "init/multiboot2.h"
 
 constexpr const size_t page_size = 4096;
 
 struct PhysicalAllocator {
-public:
     struct allocation_t {
         paddr_t ptr;
         size_t page_count;
     };
-    struct entry_t {
-        uint64_t page_count;
-        pid_t owner;
-    };
+
     static void init(const multiboot::tag_mmap* multiboot_info);
     static PhysicalAllocator& getInstance();
-    allocation_t allocPages(size_t page_count, pid_t owner);
-    void free(allocation_t ptr);
-    void force(allocation_t ptr, pid_t owner);
-    entry_t getOwner(paddr_t from);
-private:
-    struct Page {
-        optional<Page*> next;
-        entry_t entries[255];
-        void forceEntry(entry_t entry);
-        void cleanup();
-        void compact();
-    };
-    static_assert(sizeof(Page) == page_size);
-    Page* storage;    
-};
 
+    optional<allocation_t> allocPages(size_t page_count, size_t alignment_pages = 1);
+    void free(allocation_t ptr);
+    bool force(allocation_t ptr, bool free);
+
+private:
+    struct page_t {
+        page_t* next;
+        page_t* prev;
+        uint64_t count[510];
+    };
+    static_assert(sizeof(page_t) == page_size);
+
+    page_t* storage;
+
+    friend struct PhysicalAllocatorImpl;
+};

kernel/src/init/init.cpp

@@ -34,7 +34,7 @@ static void step(const char* name, T&& fn) {
     print("\n");
 }
 
-optional<paddr_t> findInitialMemoryPage(const multiboot::tag_mmap* multiboot_info) {
+optional<paddr_t> findInitialMemoryPages(const multiboot::tag_mmap* multiboot_info, size_t count) {
     const auto* base = reinterpret_cast<const uint8_t*>(multiboot_info->entries);
     const auto* end  = reinterpret_cast<const uint8_t*>(multiboot_info) + multiboot_info->size;
     for (const auto* ptr = base; ptr < end; ptr += multiboot_info->entry_size) {
@@ -42,13 +42,13 @@ optional<paddr_t> findInitialMemoryPage(const multiboot::tag_mmap* multiboot_inf
         if(entry->type != multiboot::mem_type::available) {
             continue;
         }
-        auto start_ptr = (entry->addr + page_size - 1) & (~page_size);
-        auto end_ptr = (entry->addr + entry->len) & (~page_size);
+        auto start_ptr = (entry->addr + page_size - 1) & ~(page_size - 1);
+        auto end_ptr = (entry->addr + entry->len) & ~(page_size - 1);
         auto size = end_ptr - start_ptr;
-        if(size < page_size) {
+        if(size < page_size * count) {
             continue;
         }
-        for(auto ptr = start_ptr; ptr < end_ptr; ptr += page_size) {
+        for(auto ptr = start_ptr; ptr < (end_ptr - page_size * (count - 1)); ptr += page_size) {
             if(!is_in_reserved_section(paddr_t{ptr}, page_size)) {
                 return paddr_t{ptr};
             }
@@ -59,16 +59,16 @@ optional<paddr_t> findInitialMemoryPage(const multiboot::tag_mmap* multiboot_inf
 
 void PhysicalAllocator::init(const multiboot::tag_mmap* multiboot_info) {
     PhysicalAllocator alloc;
-    auto page_opt = findInitialMemoryPage(multiboot_info);
+
+    auto page_opt = findInitialMemoryPages(multiboot_info, 1);
     if(!page_opt.has_value()) {
         panic("System could not find enough valid ram!");
     }
     auto page = page_opt.value();
     memset(page.access<uint8_t*>(), 0, page_size);
-    alloc.storage = page.access<PhysicalAllocator::Page>();
+    alloc.storage = page.access<PhysicalAllocator::page_t>();
 
-    alloc.storage->entries[0].page_count=68719476736;
-    alloc.storage->entries[0].owner=pid_reserved;
+    alloc.storage->count[0] = (1ULL << 52);
 
     step("multiboot", [&]() {
         const auto* base = reinterpret_cast<const uint8_t*>(multiboot_info->entries);
@@ -78,25 +78,28 @@ void PhysicalAllocator::init(const multiboot::tag_mmap* multiboot_info) {
             if(entry->type!=multiboot::mem_type::available) {
                 continue;
             }
-            auto start_ptr = (entry->addr + page_size - 1) & (~page_size);
-            auto end_ptr = (entry->addr + entry->len) & (~page_size);
-            auto size = end_ptr - start_ptr;
-            alloc.storage->forceEntry({size/page_size, pid_free});
+            auto start_ptr = (entry->addr + page_size - 1) & ~(page_size - 1);
+            auto end_ptr = (entry->addr + entry->len) & ~(page_size - 1);
+            if(end_ptr <= start_ptr) {
+                continue;
+            }
+            alloc.force({paddr_t{start_ptr}, (end_ptr - start_ptr) / page_size}, true);
         }
     });
+
     step("reserved_sections", [&]() {
         for_each_reserved_section([&](const KernelSection& section){
             if(section.vma_start == section.vma_end) {
                 return;
             }
-            auto start_ptr = (section.phys_start().address) & (~page_size);
-            auto end_ptr = (section.phys_end().address + page_size - 1) & (~page_size);
-            auto size = end_ptr - start_ptr;
-            alloc.storage->forceEntry({size/page_size, pid_kernel});
+            auto start_ptr = section.phys_start().address & ~(page_size - 1);
+            auto end_ptr = (section.phys_end().address + page_size - 1) & ~(page_size - 1);
+            alloc.force({paddr_t{start_ptr}, (end_ptr - start_ptr) / page_size}, false);
         });
     });
-    alloc.storage->cleanup();
-    alloc.storage->compact();
+
+    alloc.force({page, 1}, false);
+
     PhysicalAllocator::getInstance() = alloc;
 }
 

kernel/src/memory/allocator.cpp

@@ -3,14 +3,334 @@
 
 static PhysicalAllocator allocator{};
 
+struct PhysicalAllocatorImpl {
+    using page_t = PhysicalAllocator::page_t;
+    using allocation_t = PhysicalAllocator::allocation_t;
+
+    static constexpr size_t entries_per_node = 510;
+    static constexpr size_t set_range_margin = 4;
+
+    PhysicalAllocator& alloc;
+
+    struct Cursor {
+        page_t* node;
+        size_t slot;
+        uint64_t& value() { return node->count[slot]; }
+        void advance() {
+            ++slot;
+            if (slot == entries_per_node) {
+                node = node->next;
+                slot = 0;
+            }
+        }
+        void retreat() {
+            if (slot == 0) {
+                node = node->prev;
+                slot = entries_per_node - 1;
+            } else {
+                --slot;
+            }
+        }
+    };
+
+    struct run_location_t {
+        size_t index;
+        uint64_t address;
+        uint64_t length;
+    };
+    struct run_t {
+        bool is_free;
+        uint64_t length;
+    };
+    struct run_list_t {
+        run_t runs[6];
+        size_t count;
+    };
+
+    Cursor cursor_at(size_t index) {
+        Cursor cursor{alloc.storage, 0};
+        for (size_t hop = index / entries_per_node; hop > 0; --hop) {
+            cursor.node = cursor.node->next;
+        }
+        cursor.slot = index % entries_per_node;
+        return cursor;
+    }
+
+    uint64_t& entry(size_t i) {
+        return cursor_at(i).value();
+    }
+
+    size_t node_count() {
+        size_t nodes = 0;
+        for (page_t* node = alloc.storage; node; node = node->next) {
+            ++nodes;
+        }
+        return nodes;
+    }
+
+    size_t capacity() {
+        return node_count() * entries_per_node;
+    }
+
+    size_t logical_length() {
+        Cursor cursor = cursor_at(0);
+        size_t i = 0;
+        bool prev_zero = false;
+        while (true) {
+            uint64_t length = cursor.value();
+            if (length == 0 && prev_zero) {
+                return i - 1;
+            }
+            prev_zero = (length == 0);
+            cursor.advance();
+            ++i;
+        }
+    }
+
+    size_t free_slots() {
+        return capacity() - (logical_length() + 2);
+    }
+
+    optional<paddr_t> find_free_page() {
+        Cursor cursor = cursor_at(0);
+        uint64_t page = 0;
+        size_t i = 0;
+        bool prev_zero = false;
+        while (true) {
+            uint64_t length = cursor.value();
+            if (length == 0 && prev_zero) {
+                return {};
+            }
+            if ((i & 1) && length > 0) {
+                return paddr_t{page * page_size};
+            }
+            page += length;
+            prev_zero = (length == 0);
+            cursor.advance();
+            ++i;
+        }
+    }
+
+    bool ensure_free_slots(size_t needed) {
+        while (free_slots() < needed) {
+            auto free = find_free_page();
+            if (!free.has_value()) {
+                return false;
+            }
+            auto page = free.value();
+            page_t* node = page.access<page_t>();
+            memset(node, 0, page_size);
+            page_t* last = alloc.storage;
+            while (last->next) {
+                last = last->next;
+            }
+            last->next = node;
+            node->prev = last;
+            set_range(page.address / page_size, 1, false);
+        }
+        return true;
+    }
+
+    void reclaim_nodes() {
+        while (node_count() > 1) {
+            size_t capacity_without_last = (node_count() - 1) * entries_per_node;
+            if (capacity_without_last < logical_length() + 2 + set_range_margin) {
+                break;
+            }
+            page_t* prev = alloc.storage;
+            while (prev->next->next) {
+                prev = prev->next;
+            }
+            page_t* last = prev->next;
+            prev->next = nullptr;
+            uint64_t page = (reinterpret_cast<uint64_t>(last) - high_base) / page_size;
+            set_range(page, 1, true);
+        }
+    }
+
+    optional<run_location_t> locate_run(uint64_t page) {
+        Cursor cursor = cursor_at(0);
+        uint64_t address = 0;
+        size_t i = 0;
+        bool prev_zero = false;
+        while (true) {
+            uint64_t length = cursor.value();
+            if (length == 0 && prev_zero) {
+                return {};
+            }
+            if (page >= address && page < address + length) {
+                return run_location_t{i, address, length};
+            }
+            address += length;
+            prev_zero = (length == 0);
+            cursor.advance();
+            ++i;
+        }
+    }
+
+    run_list_t collect_runs(uint64_t start_page, uint64_t count, bool is_free,
+                            const run_location_t& start, const run_location_t& end, size_t total_runs) {
+        bool has_left_neighbor = start.index > 0;
+        bool has_right_neighbor = end.index + 1 < total_runs;
+        uint64_t left_remainder = start_page - start.address;
+        uint64_t right_remainder = (end.address + end.length) - (start_page + count);
+
+        run_list_t list{};
+        if (has_left_neighbor) {
+            list.runs[list.count++] = { ((start.index - 1) & 1) != 0, entry(start.index - 1) };
+        }
+        list.runs[list.count++] = { (start.index & 1) != 0, left_remainder };
+        list.runs[list.count++] = { is_free, count };
+        list.runs[list.count++] = { (end.index & 1) != 0, right_remainder };
+        if (has_right_neighbor) {
+            list.runs[list.count++] = { ((end.index + 1) & 1) != 0, entry(end.index + 1) };
+        }
+        return list;
+    }
+
+    static run_list_t merge_runs(const run_list_t& in) {
+        run_list_t out{};
+        for (size_t i = 0; i < in.count; ++i) {
+            if (in.runs[i].length == 0) {
+                continue;
+            }
+            if (out.count > 0 && out.runs[out.count - 1].is_free == in.runs[i].is_free) {
+                out.runs[out.count - 1].length += in.runs[i].length;
+            } else {
+                out.runs[out.count++] = in.runs[i];
+            }
+        }
+        return out;
+    }
+
+    static run_list_t restore_leading_parity(run_list_t list, bool first_is_free) {
+        if (list.count > 0 && list.runs[0].is_free == first_is_free) {
+            return list;
+        }
+        for (size_t i = list.count; i > 0; --i) {
+            list.runs[i] = list.runs[i - 1];
+        }
+        list.runs[0] = { first_is_free, 0 };
+        ++list.count;
+        return list;
+    }
+
+    void splice_runs(size_t first_index, size_t last_index, const run_list_t& list, size_t total_runs) {
+        size_t old_count = last_index - first_index + 1;
+        size_t terminator_end = total_runs + 2;
+        if (list.count > old_count) {
+            size_t delta = list.count - old_count;
+            Cursor source = cursor_at(terminator_end - 1);
+            Cursor dest = cursor_at(terminator_end - 1 + delta);
+            for (size_t i = terminator_end; i-- > last_index + 1; ) {
+                dest.value() = source.value();
+                source.retreat();
+                dest.retreat();
+            }
+        } else if (list.count < old_count) {
+            size_t delta = old_count - list.count;
+            Cursor source = cursor_at(last_index + 1);
+            Cursor dest = cursor_at(last_index + 1 - delta);
+            for (size_t i = last_index + 1; i < terminator_end; ++i) {
+                dest.value() = source.value();
+                source.advance();
+                dest.advance();
+            }
+            Cursor tail = cursor_at(terminator_end - delta);
+            for (size_t i = terminator_end - delta; i < terminator_end; ++i) {
+                tail.value() = 0;
+                tail.advance();
+            }
+        }
+        Cursor write = cursor_at(first_index);
+        for (size_t i = 0; i < list.count; ++i) {
+            write.value() = list.runs[i].length;
+            write.advance();
+        }
+    }
+
+    void set_range(uint64_t start_page, uint64_t count, bool is_free) {
+        if (count == 0) {
+            return;
+        }
+        auto start = locate_run(start_page);
+        auto end = locate_run(start_page + count - 1);
+        if (!start.has_value() || !end.has_value()) {
+            return;
+        }
+
+        size_t total_runs = logical_length();
+        size_t first_index = start->index > 0 ? start->index - 1 : start->index;
+        size_t last_index = end->index + 1 < total_runs ? end->index + 1 : end->index;
+
+        run_list_t collected = collect_runs(start_page, count, is_free, start.value(), end.value(), total_runs);
+        run_list_t merged = merge_runs(collected);
+        merged = restore_leading_parity(merged, (first_index & 1) != 0);
+
+        splice_runs(first_index, last_index, merged, total_runs);
+    }
+
+    bool force(allocation_t ptr, bool is_free) {
+        if (ptr.page_count == 0) {
+            return true;
+        }
+        if (!ensure_free_slots(set_range_margin)) {
+            return false;
+        }
+        set_range(ptr.ptr.address / page_size, ptr.page_count, is_free);
+        reclaim_nodes();
+        return true;
+    }
+
+    optional<allocation_t> allocPages(size_t page_count, size_t alignment_pages) {
+        if (page_count == 0) {
+            return {};
+        }
+        if (alignment_pages == 0) {
+            alignment_pages = 1;
+        }
+        if (!ensure_free_slots(set_range_margin)) {
+            return {};
+        }
+
+        Cursor cursor = cursor_at(0);
+        uint64_t run_start = 0;
+        size_t i = 0;
+        bool prev_zero = false;
+        while (true) {
+            uint64_t length = cursor.value();
+            if (length == 0 && prev_zero) {
+                break;
+            }
+            if ((i & 1) && length > 0) {
+                uint64_t aligned_start = ((run_start + alignment_pages - 1) / alignment_pages) * alignment_pages;
+                if (aligned_start + page_count <= run_start + length) {
+                    set_range(aligned_start, page_count, false);
+                    reclaim_nodes();
+                    return allocation_t{ paddr_t{aligned_start * page_size}, page_count };
+                }
+            }
+            run_start += length;
+            prev_zero = (length == 0);
+            cursor.advance();
+            ++i;
+        }
+        return {};
+    }
+};
+
 PhysicalAllocator& PhysicalAllocator::getInstance() {
     return allocator;
 }
 
-PhysicalAllocator::allocation_t PhysicalAllocator::allocPages(size_t count, pid_t owner) {
+bool PhysicalAllocator::force(allocation_t ptr, bool free) {
+    return PhysicalAllocatorImpl{*this}.force(ptr, free);
+}
 
+optional<PhysicalAllocator::allocation_t> PhysicalAllocator::allocPages(size_t page_count, size_t alignment_pages) {
+    return PhysicalAllocatorImpl{*this}.allocPages(page_count, alignment_pages);
 }
 
 void PhysicalAllocator::free(allocation_t ptr) {
-
+    PhysicalAllocatorImpl{*this}.force(ptr, true);
 }