Fix git merge problem

lib/common/browser.rb

@@ -23,6 +23,8 @@ class Browser
 
   attr_reader :hydra, :cache_dir
 
+  attr_accessor :referer
+
   # @param [ Hash ] options
   #
   # @return [ Browser ]
@@ -135,6 +137,7 @@ class Browser
       )
     end
 
+    params.merge!(referer: referer)
     params.merge!(timeout: @request_timeout) if @request_timeout
     params.merge!(connecttimeout: @connect_timeout) if @connect_timeout
 

lib/common/collections/wp_items/detectable.rb

@@ -17,6 +17,7 @@ class WpItems < Array
       hydra            = browser.hydra
       targets          = targets_items(wp_target, options)
       progress_bar     = progress_bar(targets.size, options)
+      queue_count      = 0
       exist_options    = {
         error_404_hash:  wp_target.error_404_hash,
         homepage_hash:   wp_target.homepage_hash,
@@ -43,8 +44,16 @@ class WpItems < Array
         end
 
         hydra.queue(request)
+        queue_count += 1
+
+        if queue_count >= browser.max_threads
+          hydra.run
+          queue_count = 0
+          puts "Sent #{browser.max_threads} requests ..." if options[:verbose]
+        end
       end
 
+      # run the remaining requests
       hydra.run
       results.sort!
       results # can't just return results.sort because the #sort returns an array, and we want a WpItems

lib/common/common_helper.rb

@@ -34,7 +34,7 @@ WP_VERSIONS_XSD     = DATA_DIR + '/wp_versions.xsd'
 LOCAL_FILES_XSD     = DATA_DIR + '/local_vulnerable_files.xsd'
 USER_AGENTS_FILE    = DATA_DIR + '/user-agents.txt'
 
-WPSCAN_VERSION       = '2.3'
+WPSCAN_VERSION       = '2.4'
 
 $LOAD_PATH.unshift(LIB_DIR)
 $LOAD_PATH.unshift(WPSCAN_LIB_DIR)
@@ -64,6 +64,14 @@ end
 
 require_files_from_directory(COMMON_LIB_DIR, '**/*.rb')
 
+# Hook to check if the target if down during the scan
+# The target is considered down after 10 requests with status = 0
+down = 0
+Typhoeus.on_complete do |response|
+  down += 1 if response.code == 0
+  fail 'The target seems to be down' if down >= 10
+end
+
 # Add protocol
 def add_http_protocol(url)
   url =~ /^https?:/ ? url : "http://#{url}"

lib/common/models/wp_theme/findable.rb

@@ -43,8 +43,6 @@ class WpTheme < WpItem
       end
     end
 
-    # http://code.google.com/p/wpscan/issues/detail?id=141
-    #
     # @param [ URI ] target_uri
     #
     # @return [ WpTheme ]

lib/common/models/wp_user.rb

@@ -12,7 +12,7 @@ class WpUser < WpItem
   # @return [ Array<Symbol> ]
   def allowed_options; [:id, :login, :display_name, :password] end
 
-  # @return [ URI ] The uri to the auhor page
+  # @return [ URI ] The uri to the author page
   def uri
     if id
       return @uri.merge("?author=#{id}")
@@ -54,8 +54,8 @@ class WpUser < WpItem
   # @return [ String ]
   def to_s
     s  = "#{id}"
-    s += " | #{login}" if login
-    s += " | #{display_name}" if display_name
+    s << " | #{login}" if login
+    s << " | #{display_name}" if display_name
     s
   end
 

lib/common/models/wp_version/findable.rb

@@ -190,8 +190,6 @@ class WpVersion < WpItem
 
     # Attempts to find the WordPress version from the sitemap.xml file.
     #
-    # See: http://code.google.com/p/wpscan/issues/detail?id=109
-    #
     # @param [ URI ] target_uri
     #
     # @return [ String ] The version number

lib/wpscan/web_site.rb

@@ -32,7 +32,7 @@ class WebSite
 
   def has_xml_rpc?
     response = Browser.get_and_follow_location(xml_rpc_url)
-    response.body =~ %r{XML-RPC server accepts POST requests only}i    
+    response.body =~ %r{XML-RPC server accepts POST requests only}i
   end
 
   # See http://www.hixie.ch/specs/pingback/pingback-1.0#TOC2.3
@@ -71,7 +71,7 @@ class WebSite
   #
   # @return [ String ] The MD5 hash of the page
   def self.page_hash(page)
-    page = Browser.get(page) unless page.is_a?(Typhoeus::Response)
+    page = Browser.get(page, { followlocation: true, cache_ttl: 0 }) unless page.is_a?(Typhoeus::Response)
 
     Digest::MD5.hexdigest(page.body.gsub(/<!--.*?-->/m, ''))
   end

lib/wpscan/wp_target.rb

@@ -29,6 +29,7 @@ class WpTarget < WebSite
     @multisite      = nil
 
     Browser.instance(options.merge(:max_threads => options[:threads]))
+    Browser.instance.referer = url
   end
 
   # check if the target website is
@@ -38,6 +39,11 @@ class WpTarget < WebSite
 
     response = Browser.get_and_follow_location(@uri.to_s)
 
+    # Note: in the future major WPScan version, change the user-agent to see
+    # if the response is a 200 ?
+    fail "The target is responding with a 403, this might be due to a WAF or a plugin\n" \
+          'You should try to supply a valid user-agent via the --user-agent option' if response.code == 403
+
     if response.body =~ /["'][^"']*\/wp-content\/[^"']*["']/i
       wordpress = true
     else
@@ -93,7 +99,7 @@ class WpTarget < WebSite
   end
   # :nocov:
 
-  # The version is not yet considerated
+  # The version is not yet considered
   #
   # @param [ String ] name
   # @param [ String ] version

lib/wpscan/wp_target/wp_login_protection.rb

@@ -12,7 +12,6 @@ class WpTarget < WebSite
     end
 
     # Checks if a login protection plugin is enabled
-    # http://code.google.com/p/wpscan/issues/detail?id=111
     # return a WpPlugin object or nil if no one is found
     def login_protection_plugin
       unless @login_protection_plugin

lib/wpscan/wpscan_helper.rb

@@ -101,5 +101,6 @@ def help
   puts '--max-threads <max-threads>  Maximum Threads'
   puts '--help     | -h This help screen.'
   puts '--verbose  | -v Verbose output.'
+  puts '--batch Never ask for user input, use the default behaviour.'
   puts
 end

lib/wpscan/wpscan_options.rb

@@ -3,6 +3,7 @@
 class WpscanOptions
 
   ACCESSOR_OPTIONS = [
+    :batch,
     :enumerate_plugins,
     :enumerate_only_vulnerable_plugins,
     :enumerate_all_plugins,
@@ -252,10 +253,11 @@ class WpscanOptions
       ['--basic-auth', GetoptLong::REQUIRED_ARGUMENT],
       ['--debug-output', GetoptLong::NO_ARGUMENT],
       ['--version', GetoptLong::NO_ARGUMENT],
-      ['--cache_ttl', GetoptLong::REQUIRED_ARGUMENT],
-      ['--request_timeout', GetoptLong::REQUIRED_ARGUMENT],
-      ['--connect_timeout', GetoptLong::REQUIRED_ARGUMENT],
-      ['--max_threads', GetoptLong::REQUIRED_ARGUMENT]
+      ['--cache-ttl', GetoptLong::REQUIRED_ARGUMENT],
+      ['--request-timeout', GetoptLong::REQUIRED_ARGUMENT],
+      ['--connect-timeout', GetoptLong::REQUIRED_ARGUMENT],
+      ['--max-threads', GetoptLong::REQUIRED_ARGUMENT],
+      ['--batch', GetoptLong::NO_ARGUMENT]
     )
   end
 

lib/wpstools/plugins/checker/checker_plugin.rb

@@ -32,10 +32,12 @@ class CheckerPlugin < Plugin
       xml = xml(vuln_ref_file)
 
       urls = []
-      xml.xpath('//reference').each { |node| urls << node.text }
+      xml.xpath('//references/url').each { |node| urls << node.text }
 
       urls.uniq!
 
+      puts "[!] No URLs found in #{vuln_ref_file}!" if urls.empty?
+
       dead_urls       = []
       queue_count     = 0
       request_count   = 0

lib/wpstools/plugins/stats/stats_plugin.rb

@@ -20,7 +20,6 @@ class StatsPlugin < Plugin
 
       puts "WPScan Database Statistics:"
       puts "---------------------------"
-      puts "[#] Total WordPress Sites in the World: #{get_wp_installations}"
       puts
       puts "[#] Total vulnerable versions: #{vuln_core_count}"
       puts "[#] Total vulnerable plugins:  #{vuln_plugin_count}"
@@ -79,9 +78,4 @@ class StatsPlugin < Plugin
     IO.readlines(file).size
   end
 
-  def get_wp_installations()
-    page = Nokogiri::HTML(Typhoeus.get('http://en.wordpress.com/stats/').body)
-    page.css('span[class="stats-flipper-number"]').text
-  end
-
 end