Kernel heaps: add vmem heap

Since commit 1462526 "Physical heap: switch from id heap to
buddy memory allocator", the physical memory heap does not support
allocation requests larger than 2 MB; this is done in order to keep
the heap implementation efficient in terms of both runtime
performance and memory overhead. However, there are rare cases
where the kernel needs memory allocations that exceed this limit:
for example, when booting images with large filesystems, the tuple
staging buffer for the TFS log can grow in size to several MB, and
if memory allocation fails, the kernel panics with "assertion
buffer_write(tl->tuple_staging, buffer_ref(b, 0), length) failed".
In order to allow the kernel to allocate large chunks of memory
while keeping the physical memory allocator efficient, add a new
kernel heap that supports arbitrarily large alloocations of
contiguous virtual memory which is mapped to physical memory that
can be non-contiguous. To allow booting images with large
filesystems, make the tuple staging buffer for the TFS log use this
new heap.

Closes #2105.

Author: Francesco Lavra

platform/pc/Makefile

@@ -50,6 +50,7 @@ SRCS-kernel.elf= \
 	$(SRCDIR)/kernel/storage.c \
 	$(SRCDIR)/kernel/symtab.c \
 	$(SRCDIR)/kernel/vdso-now.c \
+	$(SRCDIR)/kernel/vmem_heap.c \
 	$(SRCDIR)/net/direct.c \
 	$(SRCDIR)/net/net.c \
 	$(SRCDIR)/net/netsyscall.c \

platform/riscv-virt/Makefile

@@ -46,6 +46,7 @@ SRCS-kernel.elf= \
 	$(SRCDIR)/kernel/storage.c \
 	$(SRCDIR)/kernel/symtab.c \
 	$(SRCDIR)/kernel/vdso-now.c \
+	$(SRCDIR)/kernel/vmem_heap.c \
 	$(SRCDIR)/net/direct.c \
 	$(SRCDIR)/net/net.c \
 	$(SRCDIR)/net/netsyscall.c \

platform/virt/Makefile

@@ -72,6 +72,7 @@ SRCS-kernel.elf= \
 	$(SRCDIR)/kernel/storage.c \
 	$(SRCDIR)/kernel/symtab.c \
 	$(SRCDIR)/kernel/vdso-now.c \
+	$(SRCDIR)/kernel/vmem_heap.c \
 	$(SRCDIR)/net/direct.c \
 	$(SRCDIR)/net/net.c \
 	$(SRCDIR)/net/netsyscall.c \

src/fs/tlog.c

@@ -197,7 +197,13 @@ static log log_new(heap h, tfs fs)
     tl->dictionary = allocate_table(h, identity_key, pointer_equal);
     if (tl->dictionary == INVALID_ADDRESS)
         goto fail_dealloc_log;
-    tl->tuple_staging = allocate_buffer(h, PAGESIZE /* arbitrary */);
+    heap staging_heap;
+#ifdef KERNEL
+    staging_heap = get_kernel_heaps()->vmem;
+#else
+    staging_heap = h;
+#endif
+    tl->tuple_staging = allocate_buffer(staging_heap, PAGESIZE /* arbitrary */);
     if (tl->tuple_staging == INVALID_ADDRESS)
         goto fail_dealloc_dict;
     tl->encoding_lengths = allocate_vector(h, 512);

src/kernel/init.c

@@ -206,6 +206,9 @@ void init_kernel_heaps(void)
     heaps.malloc = locking_heap_wrapper(heaps.general, heaps.malloc);
     assert(heaps.malloc != INVALID_ADDRESS);
 
+    heaps.vmem = create_vmem_heap();
+    assert(heaps.vmem != INVALID_ADDRESS);
+
     id_heap kas_ih = create_id_heap(heaps.general, heaps.locked,
                                     kas.start, range_span(kas), PAGESIZE, true);
     assert(kas_ih != INVALID_ADDRESS);

src/kernel/kernel.h

@@ -650,6 +650,8 @@ static inline u64 phys_from_linear_backed_virt(u64 virt)
     return virt - kvmem.linear.start;
 }
 
+heap create_vmem_heap(void);
+
 void unmap_and_free_phys(u64 virtual, u64 length);
 void page_free_phys(u64 phys);
 

src/kernel/vmem_heap.c

@@ -0,0 +1,69 @@
+#include <kernel.h>
+
+typedef struct vmem_heap {
+    struct heap h;
+    heap physical;
+    heap virtual;
+} *vmem_heap;
+
+static void vmem_dealloc(heap h, u64 a, bytes len)
+{
+    bytes pagesize = h->pagesize;
+    len = pad(len, pagesize);
+    for (u64 v = a; v < a + len; v += pagesize) {
+        unmap(v, pagesize);
+    }
+    vmem_heap vmh = (vmem_heap)h;
+    deallocate(vmh->virtual, a, len);
+}
+
+static u64 vmem_alloc(heap h, bytes len)
+{
+    vmem_heap vmh = (vmem_heap)h;
+    bytes pagesize = h->pagesize;
+    len = pad(len, pagesize);
+    u64 v = allocate_u64(vmh->virtual, len);
+    if (v != INVALID_PHYSICAL) {
+        heap physical = vmh->physical;
+        pageflags flags = pageflags_writable(pageflags_memory());
+        u64 virt_offset = 0;
+        do {
+            u64 phys_len = len - virt_offset;
+            u64 p;
+            while (true) {
+                p = allocate_u64(physical, phys_len);
+                if ((p != INVALID_PHYSICAL) || (phys_len == pagesize))
+                    break;
+                phys_len = pad(phys_len >> 1, pagesize);
+            }
+            if (p != INVALID_PHYSICAL) {
+                map(v + virt_offset, p, phys_len, flags);
+                virt_offset += phys_len;
+            } else if (phys_len == pagesize) {
+                break;
+            }
+        } while (virt_offset < len);
+        if (virt_offset < len) {    /* physical memory allocation failed */
+            vmem_dealloc(h, v, virt_offset);
+            v = INVALID_PHYSICAL;
+        }
+    }
+    return v;
+}
+
+heap create_vmem_heap(void)
+{
+    kernel_heaps kh = get_kernel_heaps();
+    vmem_heap vmh = allocate(kh->locked, sizeof(*vmh));
+    if (vmh != INVALID_ADDRESS) {
+        vmh->h.alloc = vmem_alloc;
+        vmh->h.dealloc = vmem_dealloc;
+        vmh->h.allocated = 0;
+        vmh->h.total = 0;
+        vmh->h.management = 0;
+        vmh->physical = heap_physical(kh);
+        vmh->virtual = &heap_virtual_page(kh)->h;
+        vmh->h.pagesize = vmh->physical->pagesize;
+    }
+    return &vmh->h;
+}

src/runtime/kernel_heaps.h

@@ -58,6 +58,11 @@ typedef struct kernel_heaps {
 
     /* mcache for allocations of DMA memory. Protected by spinlock. */
     heap dma;
+
+    /* Heap for allocating large chunks of virtually contiguous memory that does not have to be
+     * physically contiguous. Mapping and unmapping of virtual addresses is done during allocation
+     * and deallocation. */
+    heap vmem;
 } *kernel_heaps;
 
 static inline heap heap_physical(kernel_heaps heaps)