Page cache: implement page eviction for user-mapped pages

Page cache pages that are mapped by the user process and faulted-in
on-demand are never evicted from the cache until they are unmapped;
this includes all the read-only sections of the user program ELF
(which are never unmapped).
To allow relasing memory from the above pages during low-memory or
out-of-memory conditions, enhance the memory cleaner implementation
so that it scans both shared and private mappings, and evicts "old"
pages, i.e. pages that have not been accessed recently (in OOM
conditions, be more aggressive by evicting even recently accessed
pages). To do the eviction safely on SMP machines, invoke a
synchronous (rendezvous-based) TLB shootdown.
This optimizes memory utilization by the page cache, e.g. it allows
more memory to be released to the host OS via a virtio balloon
device.

This change can cause a SIGBUS signal to be delivered to the user
process when a page cannot be faulted-in during the "ruby_alloc"
end-to-end test; the Ruby program handles this signal by dumping
process state information and then aborting (i.e. raising a SIGABRT
signal); therefore, to make this test pass, add "6" to the list of
expected exit codes for this test.

Author: francescolavra

src/aarch64/page_machine.h

@@ -407,11 +407,23 @@ static inline boolean pte_is_dirty(pte entry)
     return false;
 }
 
+static inline boolean pte_is_accessed(pte entry)
+{
+    return (entry & PAGE_ATTR_AF) != 0;
+}
+
 static inline void pt_pte_clean(pteptr pte)
 {
     // XXX TODO
 }
 
+static inline boolean pte_clear_accessed(pteptr pp)
+{
+    boolean accessed = !!(*pp & PAGE_ATTR_AF);
+    *pp &= ~PAGE_ATTR_AF;
+    return accessed;
+}
+
 static inline u64 page_from_pte(pte pte)
 {
     return pte & PAGE_4K_NEXT_TABLE_OR_PAGE_OUT_MASK;

src/kernel/init.c

@@ -688,10 +688,10 @@ closure_function(2, 3, void, attach_storage,
     apply(req_handler, &req);
 }
 
-closure_func_basic(mem_cleaner, u64, mm_pagecache_cleaner,
-                   u64 clean_bytes)
+closure_func_basic(mem_wcleaner, u64, mm_pagecache_cleaner,
+                   u64 clean_bytes, u32 flags)
 {
-    return pagecache_drain(clean_bytes);
+    return pagecache_drain(clean_bytes, flags);
 }
 
 static void read_kernel_syms(void)
@@ -731,9 +731,9 @@ void kernel_runtime_init(kernel_heaps kh)
     init_sg(locked);
     dma_init(kh);
     init_pagecache(locked, (heap)kh->pages, PAGESIZE);
-    mem_cleaner pc_cleaner = closure_func(misc, mem_cleaner, mm_pagecache_cleaner);
+    mem_wcleaner pc_cleaner = closure_func(misc, mem_wcleaner, mm_pagecache_cleaner);
     assert(pc_cleaner != INVALID_ADDRESS);
-    assert(mm_register_mem_cleaner(pc_cleaner));
+    assert(mm_register_mem_wcleaner(pc_cleaner));
     init_extra_prints();
     init_pci(kh);
     init_console(kh);

src/kernel/pagecache.c

@@ -32,6 +32,12 @@ typedef struct pagecache_page_entry {
     pagecache_page pp;
 } *pagecache_page_entry;
 
+typedef struct pagecache_drain_work {
+    buffer page_entries;
+    closure_struct(thunk, inval_complete);
+    int freed_pages;
+} *pagecache_drain_work;
+
 #define pagecache_lock_mappings()   pagecache_lock(global_pagecache)
 #define pagecache_unlock_mappings() pagecache_unlock(global_pagecache)
 
@@ -808,7 +814,76 @@ static void pagecache_delete_pages_locked(pagecache pc)
     }
 }
 
-u64 pagecache_drain(u64 drain_bytes)
+closure_function(3, 3, boolean, pagecache_check_old_page,
+                 pagecache_map, pcm, flush_entry, fe, buffer, page_entries,
+                 int level, u64 vaddr, pteptr entry)
+{
+    pagecache pc = global_pagecache;
+    pagecache_map pcm = bound(pcm);
+    pte old_entry = pte_from_pteptr(entry);
+    boolean abort = false;
+    if (pte_is_present(old_entry) && pte_is_mapping(level, old_entry) && !pte_is_dirty(old_entry)) {
+        u64 pi = (pcm->node_offset + vaddr - pcm->n.r.start) >> pc->page_order;
+        pagecache_node pn = pcm->pn;
+        pagecache_lock_node(pn);
+        pagecache_page pp = page_lookup_nodelocked(pn, pi);
+        if ((pp != INVALID_ADDRESS) && (page_from_pte(old_entry) == pp->phys)) {
+            flush_entry fe = bound(fe);
+            if (pte_clear_accessed(entry)) {
+                page_invalidate(fe, vaddr);
+                pagecache_lock_state(pc);
+                touch_page_locked(pn, pp, 0);
+                pagecache_unlock_state(pc);
+            } else if (pp->evicted) {
+                buffer page_entries = bound(page_entries);
+                if (buffer_space(page_entries) >= sizeof(struct pagecache_page_entry)) {
+                    page_invalidate(fe, vaddr);
+                    pagecache_page_entry e = buffer_end(page_entries);
+                    e->pte_ptr = entry;
+                    e->pp = pp;
+                    buffer_produce(page_entries, sizeof(*e));
+                } else {
+                    abort = true;
+                }
+            }
+        }
+        pagecache_unlock_node(pn);
+    }
+    return !abort;
+}
+
+static void pagecache_scan_old_maps(list head, flush_entry fe, buffer page_entries)
+{
+    list_foreach(head, l) {
+        pagecache_map pcm = struct_from_list(l, pagecache_map, l);
+        if (!traverse_ptes(pcm->n.r.start, range_span(pcm->n.r),
+                           stack_closure(pagecache_check_old_page, pcm, fe, page_entries)))
+            return;
+    }
+}
+
+closure_func_basic(thunk, void, pagecache_inval_complete)
+{
+    pagecache_drain_work drain_work = struct_from_closure(pagecache_drain_work, inval_complete);
+    pagecache_page_entry entry;
+    pagecache pc = global_pagecache;
+    pagecache_lock_state(pc);
+    while ((entry = buffer_pop(drain_work->page_entries, sizeof(*entry))) != 0) {
+        pteptr pte_ptr = entry->pte_ptr;
+        pagecache_page pp = entry->pp;
+        pte pt_entry = pte_from_pteptr(pte_ptr);
+        if (pte_is_present(pt_entry) && (page_from_pte(pt_entry) == pp->phys) &&
+            !pte_is_dirty(pt_entry) && !pte_is_accessed(pt_entry)) {
+            pte_set(pte_ptr, 0);
+            if (pp->refcount == 1)
+                drain_work->freed_pages++;
+            pagecache_page_release_locked(pc, pp, true);
+        }
+    }
+    pagecache_unlock_state(pc);
+}
+
+u64 pagecache_drain(u64 drain_bytes, u32 flags)
 {
     pagecache pc = global_pagecache;
     u64 pages = pad(drain_bytes, cache_pagesize(pc)) >> pc->page_order;
@@ -825,6 +900,31 @@ u64 pagecache_drain(u64 drain_bytes)
     if (drained < drain_bytes)
         drained += cache_drain((caching_heap)pc->completions, drain_bytes - drained,
                                PAGECACHE_COMPLETIONS_RETAIN * sizeof(struct page_completion));
+    if ((drained < drain_bytes) && (flags & MEMCLEAN_CANWAIT)) {
+        timestamp here = now(CLOCK_ID_MONOTONIC_RAW);
+        boolean purge_mappings;
+        if (flags & MEMCLEAN_OOM)
+            purge_mappings = true;
+        else {
+            purge_mappings = ((here - pc->map_purge) >= seconds(PAGECACHE_SCAN_PERIOD_SECONDS));
+        }
+        if (purge_mappings) {
+            pc->map_purge = here;
+            struct pagecache_drain_work drain_work;
+            buffer page_entries = drain_work.page_entries =
+                    little_stack_buffer(context_stack_space() / 2);
+            flush_entry fe = get_page_flush_entry();
+            pagecache_lock_mappings();
+            pagecache_scan_old_maps(&pc->shared_maps, fe, page_entries);
+            pagecache_scan_old_maps(&pc->private_maps, fe, page_entries);
+            pagecache_unlock_mappings();
+            drain_work.freed_pages = 0;
+            page_invalidate_sync(fe, init_closure_func(&drain_work.inval_complete, thunk,
+                                                       pagecache_inval_complete),
+                                 true);
+            drained += drain_work.freed_pages * cache_pagesize(pc);
+        }
+    }
     return drained;
 }
 
@@ -2139,6 +2239,7 @@ void init_pagecache(heap general, heap contiguous, u64 pagesize)
     list_init(&pc->private_maps);
     init_closure_func(&pc->page_compare, rb_key_compare, pagecache_page_compare);
     init_closure_func(&pc->page_print_key, rbnode_handler, pagecache_page_print_key);
+    pc->map_purge = 0;
 
     pc->writeback_in_progress = false;
     init_timer(&pc->scan_timer);

src/kernel/pagecache.h

@@ -28,7 +28,7 @@ int pagecache_get_page_order(void);
 
 u64 pagecache_get_occupancy(void);
 
-u64 pagecache_drain(u64 drain_bytes);
+u64 pagecache_drain(u64 drain_bytes, u32 flags);
 
 pagecache_node pagecache_allocate_node(pagecache_volume pv, sg_io fs_read, sg_io fs_write, pagecache_node_reserve fs_reserve);
 

src/kernel/pagecache_internal.h

@@ -34,6 +34,7 @@ typedef struct pagecache {
     struct list volumes;
     struct list shared_maps;
     struct list private_maps;
+    timestamp map_purge;
 
     boolean writeback_in_progress;
     struct timer scan_timer;

src/riscv64/page_machine.h

@@ -16,6 +16,7 @@ static inline u64 get_pagetable_base(u64 vaddr)
 #define PAGE_EXEC       U64_FROM_BIT(3)
 #define PAGE_USER       U64_FROM_BIT(4)
 #define PAGE_GLOBAL     U64_FROM_BIT(5)
+#define PAGE_ACCESSED   U64_FROM_BIT(6)
 #define PAGE_DIRTY      U64_FROM_BIT(7)
 #define PAGE_NO_BLOCK   U64_FROM_BIT(8) // RSW[0]
 #define PAGE_DEFAULT_PERMISSIONS (PAGE_READABLE)
@@ -213,11 +214,23 @@ static inline boolean pte_is_dirty(pte entry)
     return (entry & PAGE_DIRTY) != 0;
 }
 
+static inline boolean pte_is_accessed(pte entry)
+{
+    return (entry & PAGE_ACCESSED) != 0;
+}
+
 static inline void pt_pte_clean(pteptr pte)
 {
     *pte &= ~PAGE_DIRTY;
 }
 
+static inline boolean pte_clear_accessed(pteptr pp)
+{
+    boolean accessed = !!(*pp & PAGE_ACCESSED);
+    *pp &= ~PAGE_ACCESSED;
+    return accessed;
+}
+
 static inline u64 page_from_pte(pte pte)
 {
     return (pte & (MASK(54) & ~PAGE_FLAGS_MASK))<<2;

src/unix/exec.c

@@ -278,20 +278,26 @@ closure_function(4, 5, boolean, faulting_map,
         u64 vmflags = VMAP_FLAG_READABLE | VMAP_FLAG_PROG;
         if (pageflags_is_exec(flags))
             vmflags |= VMAP_FLAG_EXEC;
-        if (pageflags_is_writable(flags))
+        boolean rw = pageflags_is_writable(flags);
+        if (rw)
             vmflags |= VMAP_FLAG_WRITABLE;
         if (tail_bss > 0)
             vmflags |= VMAP_FLAG_TAIL_BSS;
         range r = irangel(map_start, data_map_size);
         exec_debug("%s: add %s to vmap: %R vmflags 0x%lx, offset 0x%lx, data_size 0x%lx, tail_bss 0x%lx\n",
                    func_ss, pageflags_is_exec(flags) ? ss("text") : ss("data"),
                    r, vmflags, offset, data_size, tail_bss);
+        pagecache_node pn = fsfile_get_cachenode(bound(f));
         struct vmap k = ivmap(vmflags, bound(allowed_flags), offset,
-                              fsfile_get_cachenode(bound(f)), 0);
+                              pn, 0);
         if (tail_bss > 0)
             k.bss_offset = data_size;
         if (allocate_vmap(bound(p), r, k) == INVALID_ADDRESS)
             goto alloc_fail;
+        if (!rw)
+            /* Make the page cache aware of this mapping so that it can evict relevant pages on
+             * memory pressure. */
+            pagecache_node_add_mapping(pn, r, offset, false);
         map_start += data_map_size;
         bss_size -= tail_bss;
     }

src/x86_64/page_machine.h

@@ -223,6 +223,11 @@ static inline boolean pte_is_dirty(pte entry)
     return (entry & PAGE_DIRTY) != 0;
 }
 
+static inline boolean pte_is_accessed(pte entry)
+{
+    return (entry & PAGE_ACCESSED) != 0;
+}
+
 static inline u64 page_from_pte(pte p)
 {
     /* page directory pointer base address [51:12] */
@@ -234,6 +239,13 @@ static inline void pt_pte_clean(pteptr pp)
     *pp &= ~PAGE_DIRTY;
 }
 
+static inline boolean pte_clear_accessed(pteptr pp)
+{
+    boolean accessed = !!(*pp & PAGE_ACCESSED);
+    *pp &= ~PAGE_ACCESSED;
+    return accessed;
+}
+
 #ifndef physical_from_virtual
 static inline u64 pte_lookup_phys(u64 table, u64 vaddr, int offset)
 {

test/e2e/ruby_alloc/config.json

@@ -7,6 +7,6 @@
     "Kernel": "../../../output/test/e2e/kernel.img",
     "ManifestPassthrough": {
         "debug_exit":"t",
-        "expected_exit_code":["9", "11"]
+        "expected_exit_code":["6", "9", "11"]
     }
 }