diff --git a/lisp/mouse.el b/lisp/mouse.el
index 3441a4787e..86427a02c6 100644
--- a/lisp/mouse.el
+++ b/lisp/mouse.el
@@ -440,42 +440,22 @@ context-menu-entry
   `(menu-item ,(purecopy "Context Menu") ignore
               :filter (lambda (_) (context-menu-map))))
 
-(defvar context-menu--saved-bindings nil
-  "Alist of bindings to restore when `context-menu-mode' is disabled.")
-
-(defun context-menu--bind-mouse (click-sym down-sym)
-  "Enable `context-menu-mode' mouse bindings.
-CLICK-SYM and DOWN-SYM are the mouse click and down key symbols to use."
-  (let ((click (vector click-sym))
-        (down (vector down-sym)))
-    (push (cons click-sym (global-key-binding click))
-          context-menu--saved-bindings)
-    (global-unset-key click)
-    (push (cons down-sym (global-key-binding down))
-          context-menu--saved-bindings)
-    (global-set-key down context-menu-entry)))
-
-(defun context-menu--restore-bindings ()
-  "Restore saved `context-menu-mode' bindings."
-  (pcase-dolist (`(,sym . ,binding) context-menu--saved-bindings)
-    (let ((key (vector sym)))
-      (if binding
-          (global-set-key key binding)
-        (global-unset-key key)))))
+(defvar context-menu-mode-map
+  (let ((map (make-sparse-keymap)))
+    (define-key map [mouse-3] nil)
+    (define-key map [down-mouse-3] context-menu-entry)
+    (when (featurep 'ns)
+      (define-key map [C-mouse-1] nil)
+      (define-key map [C-down-mouse-1] context-menu-entry))
+    map)
+  "Context-menu-mode map.")
 
 (define-minor-mode context-menu-mode
   "Toggle Context Menu mode.
 
 When Context Menu mode is enabled, clicking the mouse button down-mouse-3
 activates the menu whose contents depends on its surrounding context."
-  :global t :group 'mouse
-  (if context-menu-mode
-      (progn
-        (setq context-menu--saved-bindings nil)
-        (context-menu--bind-mouse 'mouse-3 'down-mouse-3)
-        (when (featurep 'ns)
-          (context-menu--bind-mouse 'C-mouse-1 'C-down-mouse-1)))
-    (context-menu--restore-bindings)))
+  :global t :group 'mouse)
 
 
 ;; Commands that operate on windows.