- Wordpress.com is instable

- request_timeout and connect_timeout implemented

conf/browser.conf.json

@@ -18,6 +18,8 @@
 
   "request_timeout": 2000, // 2s
 
+  "connect_timeout": 1000, // 1s
+
   "max_threads": 20,
 
   // Some user_agents can be found there http://techpatterns.com/downloads/firefox/useragentswitcher.xml (thx to Gianluca Brindisi)

lib/common/browser.rb

@@ -16,7 +16,9 @@ class Browser
     :user_agent,
     :user_agent_mode,
     :proxy,
-    :proxy_auth
+    :proxy_auth,
+    :request_timeout,
+    :connect_timeout
   ]
 
   @@instance = nil
@@ -30,7 +32,7 @@ class Browser
     @config_file = options[:config_file] || CONF_DIR + '/browser.conf.json'
     @cache_dir   = options[:cache_dir]   || CACHE_DIR + '/browser'
 
-    load_config()
+    load_config
     override_config(options)
 
     unless @hydra
@@ -70,7 +72,7 @@ class Browser
     @config_file = config_file || @config_file
 
     if File.symlink?(@config_file)
-      raise "[ERROR] Config file is a symlink."
+      raise '[ERROR] Config file is a symlink.'
     else
       data = JSON.parse(File.read(@config_file))
     end
@@ -118,6 +120,14 @@ class Browser
       )
     end
 
+    if @request_timeout
+      params = params.merge(timeout: @request_timeout)
+    end
+
+    if @connect_timeout
+      params = params.merge(connecttimeout: @connect_timeout)
+    end
+
     # Used to enable the cache system if :cache_ttl > 0
     unless params.has_key?(:cache_ttl)
       params = params.merge(cache_ttl: @cache_ttl)

lib/common/browser/options.rb

@@ -5,7 +5,7 @@ class Browser
 
     USER_AGENT_MODES = %w{ static semi-static random }
 
-    attr_accessor :available_user_agents, :cache_ttl
+    attr_accessor :available_user_agents, :cache_ttl, :request_timeout, :connect_timeout
     attr_reader   :basic_auth, :user_agent_mode, :proxy, :proxy_auth
     attr_writer   :user_agent
 
@@ -115,6 +115,22 @@ class Browser
       end
     end
 
+    # Sets the request timeout
+    # @param [ Integer ] timeout Timeout in ms
+    #
+    # @return [ void ]
+    def request_timeout=(timeout)
+      @request_timeout = timeout
+    end
+
+    # Sets the connect timeout
+    # @param [ Integer ] timeout Timeout in ms
+    #
+    # @return [ void ]
+    def connect_timeout=(timeout)
+      @connect_timeout = timeout
+    end
+
     protected
 
     def invalid_proxy_auth_format

lib/wpstools/plugins/list_generator/generate_list.rb

@@ -11,17 +11,17 @@ class GenerateList
       @type           = 'plugin'
       @svn_url        = 'http://plugins.svn.wordpress.org/'
       @popular_url    = 'http://wordpress.org/plugins/browse/popular/'
-      @popular_regex  = %r{<h3><a href="http://wordpress.org/plugins/(.+)/">.+</a></h3>}i
+      @popular_regex  = %r{<h3><a href="http://wordpress.org/plugins/([^/]+)/">.+</a></h3>}i
     elsif type =~ /themes/i
       @type           = 'theme'
       @svn_url        = 'http://themes.svn.wordpress.org/'
       @popular_url    = 'http://wordpress.org/themes/browse/popular/'
-      @popular_regex  = %r{<h3><a href="http://wordpress.org/themes/(.+)">.+</a></h3>}i
+      @popular_regex  = %r{<h3><a href="http://wordpress.org/themes/([^/]+)">.+</a></h3>}i
     else
       raise "Type #{type} not defined"
     end
     @verbose  = verbose
-    @browser  = Browser.instance
+    @browser  = Browser.instance(request_timeout: 20000, connect_timeout: 20000, max_threads: 1)
     @hydra    = @browser.hydra
   end
 
@@ -77,12 +77,20 @@ class GenerateList
       queue_count += 1
 
       request.on_complete do |response|
+        if response.code != 200
+          puts red("Got HTTP Status #{response.code} for page #{page}. Retrying request...")
+          # Retry
+          @hydra.queue(request)
+          next
+        end
         puts "[+] Parsing page #{page_count}" if @verbose
         page_count += 1
+        found = 0
         response.body.scan(@popular_regex).each do |item|
-          puts "[+] Found popular #@type: #{item}" if @verbose
           found_items << item[0]
+          found = found + 1
         end
+        puts "[+] Found #{found} items on page #{page}" if @verbose
       end
 
       @hydra.queue(request)

lib/wpstools/plugins/stats/stats_plugin.rb

@@ -12,8 +12,8 @@ class StatsPlugin < Plugin
 
   def run(options = {})
     if options[:stats]
-      puts "Wpscan Databse Statistics:"
+      puts 'Wpscan Databse Statistics:'
-      puts "--------------------------"
+      puts '--------------------------'
       puts "[#] Total vulnerable plugins: #{vuln_plugin_count}"
       puts "[#] Total vulnerable themes: #{vuln_theme_count}"
       puts "[#] Total plugin vulnerabilities: #{plugin_vulns_count}"
@@ -25,19 +25,19 @@ class StatsPlugin < Plugin
   end
 
   def vuln_plugin_count(file=PLUGINS_VULNS_FILE)
-    xml(file).xpath("count(//plugin)").to_i
+    xml(file).xpath('count(//plugin)').to_i
   end
 
   def vuln_theme_count(file=THEMES_VULNS_FILE)
-    xml(file).xpath("count(//theme)").to_i
+    xml(file).xpath('count(//theme)').to_i
   end
 
   def plugin_vulns_count(file=PLUGINS_VULNS_FILE)
-    xml(file).xpath("count(//vulnerability)").to_i
+    xml(file).xpath('count(//vulnerability)').to_i
   end
 
   def theme_vulns_count(file=THEMES_VULNS_FILE)
-    xml(file).xpath("count(//vulnerability)").to_i
+    xml(file).xpath('count(//vulnerability)').to_i
   end
 
   def total_plugins(file=PLUGINS_FULL_FILE)