diff --git a/Library/Homebrew/cmd/fetch.rb b/Library/Homebrew/cmd/fetch.rb
index cb86545aeb..9b0c1389cb 100644
--- a/Library/Homebrew/cmd/fetch.rb
+++ b/Library/Homebrew/cmd/fetch.rb
@@ -60,19 +60,15 @@ module Homebrew
   private
 
   def retry_fetch? f
-    @fetch_failed ||= {}
-    already_failed = @fetch_failed.fetch(f.name, false)
-
-    if already_failed || !ARGV.include?("--retry")
+    @fetch_failed ||= Set.new
+    if ARGV.include?("--retry") && @fetch_failed.add?(f.name)
+      ohai "Retrying download"
+      f.clear_cache
+      true
+    else
       Homebrew.failed = true
-      return false
+      false
     end
-
-    ohai "Retrying download"
-
-    f.clear_cache
-    @fetch_failed[f.name] = true
-    true
   end
 
   def fetch_fetchable f