diff --git a/texmathp.el b/texmathp.el
index 350406b..d449f5f 100644
--- a/texmathp.el
+++ b/texmathp.el
@@ -171,7 +171,7 @@ customize (customize calls it when setting the variable)."
 		  (mapconcat 'regexp-quote switches "\\|")
 		  "\\)")
 	  texmathp-toggle-regexp
-	  (concat "\\([^\\\\\\$]\\|\\`\\)\\("
+	  (concat "\\("
 		  (mapconcat 'regexp-quote togglers "\\|")
 		  "\\)"))))
 
@@ -285,9 +285,13 @@ See the variable `texmathp-tex-commands' about which commands are checked."
 	(save-excursion
 	  (goto-char (cdr match))
 	  (while (re-search-forward texmathp-toggle-regexp pos t)
-	    (if (setq math-on (not math-on))
-		(setq sw-match (cons (match-string 2) (match-beginning 2)))
-	      (setq sw-match nil)))
+	    ;; Make sure the togglers are unescaped.
+	    (if (save-excursion
+		  (goto-char (match-beginning 0))
+		  (zerop (mod (skip-chars-backward "\\\\") 2)))
+		(if (setq math-on (not math-on))
+		    (setq sw-match (cons (match-string 1) (match-beginning 1)))
+		  (setq sw-match nil))))
 	  (and math-on sw-match (setq match sw-match))))
 
     ;; Store info, show as message when interactive, and return