diff --git a/plugins/history-substring-search/history-substring-search.plugin.zsh b/plugins/history-substring-search/history-substring-search.plugin.zsh
index 25fd3a2da..4c3d30638 100644
--- a/plugins/history-substring-search/history-substring-search.plugin.zsh
+++ b/plugins/history-substring-search/history-substring-search.plugin.zsh
@@ -15,9 +15,13 @@ fi
 # Bind terminal-specific up and down keys
 
 if [[ -n "$terminfo[kcuu1]" ]]; then
-  bindkey "$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M emacs "$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M viins "$terminfo[kcuu1]" history-substring-search-up
+  bindkey -M vicmd "$terminfo[kcuu1]" history-substring-search-up
 fi
 if [[ -n "$terminfo[kcud1]" ]]; then
-  bindkey "$terminfo[kcud1]" history-substring-search-down
+  bindkey -M emacs "$terminfo[kcud1]" history-substring-search-down
+  bindkey -M viins "$terminfo[kcud1]" history-substring-search-down
+  bindkey -M vicmd "$terminfo[kcud1]" history-substring-search-down
 fi