Physical heap: switch from id heap to buddy memory allocator

This change introduces a new heap type that implements a buddy
memory allocator and is now used as physical memory heap; this
brings the following improvements:
- lower memory fragmentation: the buddy allocator keeps track of
free contiguous memory areas of each power-of-2 size, and when
allocating a requested size, uses a minimum-sized free area to
satisfy the request
- higher memory re-use (which maximizes TLB hit rate and minimizes
host memory usage): when there are multiple minimum-sized memory
areas available to satisfy an allocation request, the buddy
allocator selects the most recently used area
- higher scalability: the buddy allocator does not rely on linear
searches on a given memory range to select a free area; therefore,
the average CPU use to execute allocation requests does not
increase with the size of physical memory

The typical overhead of the buddy allocator is around 0.1%, i.e.
99.9% of the physical memory available for a VM can be used for
kernel and user program allocations.
The size of the bootstrap heap does no longer depend on the size
of physical memory, therefore calculation of the memory size is no
longer done when unneeded, and the bootstrap heap is set up in
common code instead of platform-specific code.
The PAGEHEAP_LOWMEM_PAGESIZE constant now sets the upper limit for
physical memory allocation requests on low-memory guests, and has
been changed to 1 MB to be able to allocate the async queue used by
the scheduler.
The `pages` kernel heap is now a simple wrapper (whose purpose is
to avoid complete exhaustion of physical memory) around the
page-backed heap, and is used by both the pagecache and the mmap
code.

Author: francescolavra

platform/pc/service.c

@@ -121,7 +121,7 @@ void reclaim_regions(void)
     for_regions(e) {
         if (e->type == REGION_RECLAIM) {
             unmap(e->base, e->length);
-            if (!id_heap_add_range(heap_physical(get_kernel_heaps()), e->base, e->length))
+            if (!pageheap_add_range(e->base, e->length))
                 halt("%s: add range for physical heap failed (%R)\n",
                      func_ss, irange(e->base, e->base + e->length));
         }
@@ -262,9 +262,9 @@ static void find_initial_pages(void)
     halt("no initial pages region found; halt\n");
 }
 
-id_heap init_physical_id_heap(heap h)
+void init_physical_heap(void)
 {
-    u64 phys_length = 0;
+    /* Carve the bootstrap heap out of a physical memory region. */
     for_regions(e) {
         if (e->type == REGION_PHYSICAL) {
             /* Remove low memory area from physical memory regions, so that it can be used for
@@ -279,27 +279,18 @@ id_heap init_physical_id_heap(heap h)
                 }
             }
 
-            phys_length += e->length;
-        }
-    }
-    u64 bootstrap_size = init_bootstrap_heap(phys_length);
-
-    /* Carve the bootstrap heap out of a physical memory region. */
-    for_regions(e) {
-        if (e->type == REGION_PHYSICAL) {
             u64 base = pad(e->base, PAGESIZE);
             u64 end = e->base + e->length;
             u64 length = (end & ~MASK(PAGELOG)) - base;
-            if (length >= bootstrap_size) {
-                map(BOOTSTRAP_BASE, base, bootstrap_size, pageflags_writable(pageflags_memory()));
-                e->base = base + bootstrap_size;
+            if (length >= BOOTSTRAP_SIZE) {
+                map(BOOTSTRAP_BASE, base, BOOTSTRAP_SIZE, pageflags_writable(pageflags_memory()));
+                e->base = base + BOOTSTRAP_SIZE;
                 e->length = end - e->base;
                 break;
             }
         }
     }
 
-    id_heap physical = allocate_id_heap(h, h, PAGESIZE, true);
     boolean found = false;
     early_init_debug("physical memory:");
     for_regions(e) {
@@ -315,15 +306,14 @@ id_heap init_physical_id_heap(heap h)
 	    early_debug_u64(base + length);
 	    early_debug(")\n");
 #endif
-	    if (!id_heap_add_range(physical, base, length))
+	    if (!pageheap_add_range(base, length))
 		halt("    - id_heap_add_range failed\n");
 	    found = true;
 	}
     }
     if (!found) {
 	halt("no valid physical regions found; halt\n");
     }
-    return physical;
 }
 
 static void setup_initmap(void)

platform/riscv-virt/service.c

@@ -25,9 +25,9 @@ u64 machine_random_seed(void)
 
 extern void *START, *END;
 
-id_heap init_physical_id_heap(heap h)
+void init_physical_heap(void)
 {
-    init_debug("init_physical_id_heap\n");
+    init_debug("init_physical_heap\n");
     u64 kernel_size = pad(u64_from_pointer(&END) -
                           u64_from_pointer(&START), PAGESIZE);
 
@@ -45,20 +45,17 @@ id_heap init_physical_id_heap(heap h)
 
     u64 base = KERNEL_PHYS + kernel_size;
     u64 end = PHYSMEM_BASE + mem_size;
-    u64 bootstrap_size = init_bootstrap_heap(end - base);
-    map(BOOTSTRAP_BASE, base, bootstrap_size, pageflags_writable(pageflags_memory()));
-    base += bootstrap_size;
+    map(BOOTSTRAP_BASE, base, BOOTSTRAP_SIZE, pageflags_writable(pageflags_memory()));
+    base += BOOTSTRAP_SIZE;
     init_debug("\nfree base ");
     init_debug_u64(base);
     init_debug("\nend ");
     init_debug_u64(end);
     init_debug("\n");
-    id_heap physical = allocate_id_heap(h, h, PAGESIZE, true);
-    if (!id_heap_add_range(physical, base, end - base)) {
-        halt("init_physical_id_heap: failed to add range %R\n",
+    if (!pageheap_add_range(base, end - base)) {
+        halt("init_physical_heap: failed to add range %R\n",
              irange(base, end));
     }
-    return physical;
 }
 
 range kern_get_elf(void)

platform/virt/service.c

@@ -69,33 +69,24 @@ static void uefi_mem_map_iterate(uefi_mem_map mem_map, range_handler h)
     }
 }
 
-closure_function(1, 1, boolean, get_mem_size,
-                 u64 *, mem_size,
+closure_function(2, 1, boolean, get_bootstrap_base,
+                 range, rsvd, u64 *, base,
                  range r)
 {
-    *bound(mem_size) += range_span(r);
-    return true;
-}
-
-closure_function(3, 1, boolean, get_bootstrap_base,
-                 range, rsvd, u64, bootstrap_size, u64 *, base,
-                 range r)
-{
-    u64 bootstrap_size = bound(bootstrap_size);
     range r1, r2;
     range_difference(r, bound(rsvd), &r1, &r2);
-    if (range_span(r1) >= bootstrap_size) {
+    if (range_span(r1) >= BOOTSTRAP_SIZE) {
         *bound(base) = r1.start;
         return false;
     }
-    if (range_span(r2) >= bootstrap_size) {
+    if (range_span(r2) >= BOOTSTRAP_SIZE) {
         *bound(base) = r2.start;
         return false;
     }
     return true;
 }
 
-static void add_heap_range_internal(id_heap h, range r, range *remainder)
+static void add_heap_range_internal(range r, range *remainder)
 {
     if (remainder) {
         if (range_empty(*remainder)) {
@@ -121,31 +112,30 @@ static void add_heap_range_internal(id_heap h, range r, range *remainder)
     init_debug(" 0x");
     init_debug_u64(r.end);
     init_debug(")\n");
-    id_heap_add_range(h, r.start, range_span(r));
+    pageheap_add_range(r.start, range_span(r));
 }
 
-static inline void add_heap_range_helper(id_heap h, range r, range rsvd, range *remainder)
+static inline void add_heap_range_helper(range r, range rsvd, range *remainder)
 {
     if (!range_empty(r)) {
         range r1, r2;
         range_difference(r, rsvd, &r1, &r2);
         if (!range_empty(r1))
-            add_heap_range_internal(h, r1, remainder);
+            add_heap_range_internal(r1, remainder);
         if (!range_empty(r2))
-            add_heap_range_internal(h, r2, remainder);
+            add_heap_range_internal(r2, remainder);
     }
 }
 
-closure_function(4, 1, boolean, add_heap_range,
-                 id_heap, h, range, rsvd1, range, rsvd2, range *, remainder,
+closure_function(3, 1, boolean, add_heap_range,
+                 range, rsvd1, range, rsvd2, range *, remainder,
                  range r)
 {
-    id_heap h = bound(h);
     range *remainder = bound(remainder);
     range r1, r2;
     range_difference(r, bound(rsvd1), &r1, &r2);
-    add_heap_range_helper(h, r1, bound(rsvd2), remainder);
-    add_heap_range_helper(h, r2, bound(rsvd2), remainder);
+    add_heap_range_helper(r1, bound(rsvd2), remainder);
+    add_heap_range_helper(r2, bound(rsvd2), remainder);
     return true;
 }
 
@@ -158,63 +148,51 @@ static u64 get_memory_size(void *dtb)
 }
 
 extern void *START, *END;
-id_heap init_physical_id_heap(heap h)
+void init_physical_heap(void)
 {
-    init_debug("init_physical_id_heap\n");
+    init_debug("init_physical_heap\n");
     u64 kernel_size = pad(u64_from_pointer(&END) -
                           u64_from_pointer(&START), PAGESIZE);
 
     init_debug("init_setup_stack: kernel size ");
     init_debug_u64(kernel_size);
 
-    id_heap physical;
     if (boot_params.mem_map.map) {
         u64 map_base = u64_from_pointer(boot_params.mem_map.map);
         u64 map_size = pad((map_base & PAGEMASK) + boot_params.mem_map.map_size, PAGESIZE);
         map_base &= ~PAGEMASK;
         /* map_base has been identity-mapped in ueft_rt_init_virt() */
-        u64 mem_size = 0;
-        uefi_mem_map_iterate(&boot_params.mem_map, stack_closure(get_mem_size, &mem_size));
-        init_debug("\nmem size ");
-        init_debug_u64(mem_size);
-        u64 bootstrap_size = init_bootstrap_heap(mem_size);
         range reserved = irange(DEVICETREE_BLOB_BASE + kernel_phys_offset,
                                 KERNEL_PHYS + kernel_size + kernel_phys_offset);
         u64 base = 0;
         uefi_mem_map_iterate(&boot_params.mem_map,
-                             stack_closure(get_bootstrap_base, reserved, bootstrap_size, &base));
+                             stack_closure(get_bootstrap_base, reserved, &base));
         init_debug("\nbootstrap base ");
         init_debug_u64(base);
-        init_debug(", size ");
-        init_debug_u64(bootstrap_size);
         init_debug("\n");
         assert(!(base & PAGEMASK));
-        map(BOOTSTRAP_BASE, base, bootstrap_size, pageflags_writable(pageflags_memory()));
-        physical = allocate_id_heap(h, h, PAGESIZE, true);
+        map(BOOTSTRAP_BASE, base, BOOTSTRAP_SIZE, pageflags_writable(pageflags_memory()));
         range remainder = irange(0, 0);
-        uefi_mem_map_iterate(&boot_params.mem_map, stack_closure(add_heap_range, physical, reserved,
-                                                                 irangel(base, bootstrap_size),
+        uefi_mem_map_iterate(&boot_params.mem_map, stack_closure(add_heap_range, reserved,
+                                                                 irangel(base, BOOTSTRAP_SIZE),
                                                                  &remainder));
-        add_heap_range_internal(physical, remainder, 0);
+        add_heap_range_internal(remainder, 0);
         unmap(map_base, map_size);
     } else {
         u64 base = KERNEL_PHYS + kernel_size;
         u64 end = PHYSMEM_BASE + get_memory_size(pointer_from_u64(DEVICETREE_BLOB_BASE));
-        u64 bootstrap_size = init_bootstrap_heap(end - base);
-        map(BOOTSTRAP_BASE, base, bootstrap_size, pageflags_writable(pageflags_memory()));
-        base += bootstrap_size;
+        map(BOOTSTRAP_BASE, base, BOOTSTRAP_SIZE, pageflags_writable(pageflags_memory()));
+        base += BOOTSTRAP_SIZE;
         init_debug("\nfree base ");
         init_debug_u64(base);
         init_debug("\nend ");
         init_debug_u64(end);
         init_debug("\n");
-        physical = allocate_id_heap(h, h, PAGESIZE, true);
-        if (!id_heap_add_range(physical, base, end - base)) {
-            halt("init_physical_id_heap: failed to add range %R\n",
+        if (!pageheap_add_range(base, end - base)) {
+            halt("init_physical_heap: failed to add range %R\n",
                  irange(base, end));
         }
     }
-    return physical;
 }
 
 range kern_get_elf(void)
@@ -255,7 +233,7 @@ static void ueft_rt_init_virt(void)
     u64 map_size = pad((map_base & PAGEMASK) + mem_map->map_size, PAGESIZE);
     map_base &= ~PAGEMASK;
     pageflags flags = pageflags_writable(pageflags_memory());
-    map(map_base, map_base, map_size, flags);   /* will be unmapped in init_physical_id_heap() */
+    map(map_base, map_base, map_size, flags);   /* will be unmapped in init_physical_heap() */
     int num_desc = mem_map->map_size / mem_map->desc_size;
     u64 rt_svc_offset = 0;
     for (int i = 0; i < num_desc; i++) {
@@ -490,7 +468,7 @@ void init_platform_devices(kernel_heaps kh)
     vector cpu_ids = cpus_init_ids(heap_general(kh));
     platform_dtb_parse(kh, cpu_ids);
     /* the device tree blob is never accessed from now on: reclaim the memory where it is located */
-    id_heap_add_range(heap_physical(kh), DEVICETREE_BLOB_BASE + kernel_phys_offset,
+    pageheap_add_range(DEVICETREE_BLOB_BASE + kernel_phys_offset,
                       INIT_PAGEMEM - DEVICETREE_BLOB_BASE);
     struct console_driver *console_driver = 0;
     init_acpi_tables(kh);

src/config.h

@@ -67,7 +67,7 @@
 #define PAGECACHE_SCAN_PERIOD_SECONDS 5
 #define PAGEHEAP_MEMORY_RESERVE         (8 * MB)
 #define PAGEHEAP_LOWMEM_MEMORY_RESERVE  (4 * MB)
-#define PAGEHEAP_LOWMEM_PAGESIZE        (128*KB)
+#define PAGEHEAP_LOWMEM_PAGESIZE        (1 * MB)
 #define LOW_MEMORY_THRESHOLD   (64 * MB)
 #define SG_FRAG_BYTE_THRESHOLD (128*KB)
 #define PAGECACHE_MAX_SG_ENTRIES    8192

src/kernel/init.c

@@ -56,21 +56,6 @@ static u64 bootstrap_alloc(heap h, bytes length)
 BSS_RO_AFTER_INIT static struct kernel_heaps heaps;
 BSS_RO_AFTER_INIT static vector shutdown_completions;
 
-u64 init_bootstrap_heap(u64 phys_length)
-{
-    u64 page_count = phys_length >> PAGELOG;
-
-    /* In theory, when initializing the physical heap, the bootstrap heap must accommodate 1 bit per
-     * physical memory page (as needed by the id heap bitmap); but due to the way buffer extension
-     * works, when an id heap is pre-allocated, its bitmap allocates twice the amount of memory
-     * needed; thus, the bootstrap heap needs twice the theoretical amount of memory.
-     * In addition, we need some extra space for various initial allocations. */
-    u64 bootstrap_size = 8 * PAGESIZE + pad(page_count >> 2, PAGESIZE);
-
-    bootstrap_limit = BOOTSTRAP_BASE + bootstrap_size;
-    return bootstrap_size;
-}
-
 /* Kernel address space layout randomization.
  * Functions that call (directly or indirectly) this function must either not return to their
  * caller, or execute `return_offset(kas_kern_offset - kernel_phys_offset)` before returning to
@@ -115,9 +100,10 @@ void init_kernel_heaps(void)
     BSS_RO_AFTER_INIT static struct heap bootstrap;
     bootstrap.alloc = bootstrap_alloc;
     bootstrap.dealloc = leak;
+    bootstrap_limit = BOOTSTRAP_BASE + BOOTSTRAP_SIZE;
 
-    heaps.physical = init_physical_id_heap(&bootstrap);
-    assert(heaps.physical != INVALID_ADDRESS);
+    heaps.physical = pageheap_init(&bootstrap);
+    init_physical_heap();
 
     heaps.linear_backed = allocate_linear_backed_heap(&bootstrap, heaps.physical, irange(0, 0));
 #if defined(MEMDEBUG_BACKED) || defined(MEMDEBUG_ALL)
@@ -138,18 +124,19 @@ void init_kernel_heaps(void)
     heaps.virtual_huge = create_id_heap(&bootstrap, &bootstrap, kmem_base,
                                         KMEM_LIMIT - kmem_base, HUGE_PAGESIZE, true);
 
-    /* Pre-allocate all memory that might be needed for the physical and virtual huge heaps, so that
+    /* Pre-allocate all memory that might be needed for the virtual huge heap, so that
      * during runtime all allocations on the bootstrap heap come from a single source protected by a
      * lock (i.e. the virtual page heap). */
-    id_heap_prealloc(heaps.physical);
     id_heap_prealloc(heaps.virtual_huge);
 
     heaps.virtual_page = create_id_heap_backed(&bootstrap, &bootstrap,
                                                (heap)heaps.virtual_huge, PAGESIZE, true);
+    u64 virt_base;
     boolean kernmem_equals_dmamem = (pageflags_kernel_data().w == pageflags_dma().w);
     if (kernmem_equals_dmamem) {
         heaps.page_backed = heaps.linear_backed;
         init_page_tables((heap)heaps.physical);
+        virt_base = LINEAR_BACKED_BASE;
     } else {
         /* The linear_backed heap cannot be used for non-DMA kernel data, thus we need another
          * linear mapping for the page tables: do this mapping, then use it to create the
@@ -159,15 +146,14 @@ void init_kernel_heaps(void)
 #if defined(MEMDEBUG_BACKED) || defined(MEMDEBUG_ALL)
         heaps.page_backed = mem_debug_backed(&bootstrap, heaps.page_backed, PAGESIZE_2M, true);
 #endif
+        virt_base = mapped_virt.start;
     }
 
     boolean is_lowmem = is_low_memory_machine();
+    pageheap_init_done(pointer_from_u64(virt_base),
+                       is_lowmem ? PAGEHEAP_LOWMEM_PAGESIZE : PAGESIZE_2M);
     u64 memory_reserve = is_lowmem ? PAGEHEAP_LOWMEM_MEMORY_RESERVE : PAGEHEAP_MEMORY_RESERVE;
-    heaps.pages = allocate_objcache(&bootstrap,
-                                    reserve_heap_wrapper(&bootstrap, (heap)heaps.page_backed,
-                                                         memory_reserve),
-                                    PAGESIZE, is_lowmem ? PAGEHEAP_LOWMEM_PAGESIZE : PAGESIZE_2M,
-                                    true);
+    heaps.pages = reserve_heap_wrapper(&bootstrap, (heap)heaps.page_backed, memory_reserve);
     int max_mcache_order = is_lowmem ? MAX_LOWMEM_MCACHE_ORDER : MAX_MCACHE_ORDER;
     bytes pagesize = is_lowmem ? U64_FROM_BIT(max_mcache_order + 1) : PAGESIZE_2M;
     heaps.general = allocate_mcache(&bootstrap, (heap)heaps.page_backed, 5, max_mcache_order,
@@ -441,10 +427,6 @@ static u64 mm_clean(u64 clean_bytes)
     }
     spin_unlock(&mm_lock);
     u64 cleaned = clean_bytes - remain;
-    if (cleaned)
-        /* Memory cleaners may have deallocated page heap memory: drain the page heap, so that
-         * deallocated memory can be returned to the physical heap. */
-        cache_drain(init_heaps->pages, cleaned, 0);
     return cleaned;
 }
 

src/kernel/kernel.h

@@ -597,10 +597,10 @@ static inline boolean sched_queue_empty(sched_queue sq)
 #define BREAKPOINT_READ_WRITE 11
 
 #define BOOTSTRAP_BASE  KMEM_BASE
+#define BOOTSTRAP_SIZE  (8 * PAGESIZE)
 
 void kaslr(void);
-u64 init_bootstrap_heap(u64 phys_length);
-id_heap init_physical_id_heap(heap h);
+void init_physical_heap(void);
 void init_kernel_heaps(void);
 void init_platform_devices(kernel_heaps kh);
 void init_cpuinfo_machine(cpuinfo ci, heap backed);
@@ -635,7 +635,7 @@ backed_heap allocate_page_backed_heap(heap meta, heap virtual, heap physical,
                                       u64 pagesize, boolean locking);
 void page_backed_dealloc_virtual(backed_heap bh, u64 x, bytes length);
 
-backed_heap allocate_linear_backed_heap(heap meta, id_heap physical, range mapped_virt);
+backed_heap allocate_linear_backed_heap(heap meta, heap physical, range mapped_virt);
 
 static inline boolean is_linear_backed_address(u64 address)
 {