diff --git a/plugins/ssh-agent/ssh-agent.plugin.zsh b/plugins/ssh-agent/ssh-agent.plugin.zsh
index a688855d0..2a860f3aa 100644
--- a/plugins/ssh-agent/ssh-agent.plugin.zsh
+++ b/plugins/ssh-agent/ssh-agent.plugin.zsh
@@ -12,16 +12,28 @@ function _start_agent() {
 }
 
 function _add_identities() {
-	local id line
-	local -a identities ids
+	local id line sig
+	local -a identities loaded signatures
 	zstyle -a :omz:plugins:ssh-agent identities identities
 
-	# get list of loaded identities
-	for line in ${(f)"$(ssh-add -l)"}; do ids+=${${(z)line}[3]}; done
+	# check for .ssh folder presence
+	if [[ ! -d $HOME/.ssh ]]; then
+		return
+	fi
+
+	# get list of loaded identities' signatures
+	for line in ${(f)"$(ssh-add -l)"}; do loaded+=${${(z)line}[2]}; done
+
+	# get signatures of private keys
+	for id in $identities; do
+		signatures+="$(ssh-keygen -lf "$HOME/.ssh/$id" | awk '{print $2}')	$id"
+	done
 
 	# add identities if not already loaded
-	for id in ${^identities}; do
-		[[ ${ids[(I)$HOME/.ssh/$id]} -le 0 ]] && ssh-add $HOME/.ssh/$id
+	for sig in $signatures; do
+		id="$(cut -f2 <<< $sig)"
+		sig="$(cut -f1 <<< $sig)"
+		[[ ${loaded[(I)$sig]} -le 0 ]] && ssh-add $HOME/.ssh/$id
 	done
 }