diff --git a/sys/dev/drm/radeon/radeon_vce.c b/sys/dev/drm/radeon/radeon_vce.c
--- a/sys/dev/drm/radeon/radeon_vce.c
+++ b/sys/dev/drm/radeon/radeon_vce.c
@@ -677,7 +677,8 @@
 		 * handle and got an error, anyway free the handle
 		 */
 		for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i)
-			atomic_cmpxchg(&p->rdev->vce.handles[i], handle, 0);
+			if (p->rdev->vce.filp[i] == p->filp)
+				atomic_cmpxchg(&p->rdev->vce.handles[i], handle, 0);
 	}
 
 	return r;