--- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -682,17 +682,28 @@ p->p_usched->release_curproc(lp); /* - * Destroy the per-thread shared page and remove from any pmaps - * it resides in. + * Flag the LWP as exiting BEFORE tearing down its lpmap. + * + * lwp_usermap() (kern_proc.c) refuses to install an lpmap into an + * LWP with LWP_MP_WEXIT set, re-checking the flag after its + * (potentially sleeping) allocation while holding lwp_token. + * lwp_userunmap() saves and NULLs lwp_lpmap under the same token. + * Without WEXIT set here first, a concurrent /dev/lpmap page fault + * (user_kernel_mapping(), kern_memio.c) that calls lwp_usermap() + * while lwp_userunmap() is parked in its vm_map_remove() drain can + * install a fresh lpmap AFTER the old one was saved - the new + * allocation is then never freed (lwp_userunmap() is the only free + * site), leaking 4KB of M_UPMAP kernel heap per race hit, and the + * LWP is reaped with the mapping still installed. */ - lwp_userunmap(lp); + ASSERT_LWKT_TOKEN_HELD(&p->p_token); + atomic_set_int(&lp->lwp_mpflags, LWP_MP_WEXIT); /* - * lwp_exit() may be called without setting LWP_MP_WEXIT, so - * make sure it is set here. + * Destroy the per-thread shared page and remove from any pmaps + * it resides in. */ - ASSERT_LWKT_TOKEN_HELD(&p->p_token); - atomic_set_int(&lp->lwp_mpflags, LWP_MP_WEXIT); + lwp_userunmap(lp); /* * Clean up any virtualization