-# File lib/common/browser.rb, line 50 +# File lib/common/browser.rb, line 52 def self.instance(options = {}) unless @@instance @@instance = new(options) @@ -463,7 +463,7 @@+ + @@ -488,6 +526,41 @@ Basic base_64_encoded-# File lib/common/browser.rb, line 57 +# File lib/common/browser.rb, line 59 def self.reset @@instance = nil end@@ -504,7 +504,7 @@+-# File lib/common/browser.rb, line 90 +# File lib/common/browser.rb, line 92 def forge_request(url, params = {}) Typhoeus::Request.new(url, merge_request_params(params)) end@@ -544,12 +544,12 @@ href="Browser.html#attribute-i-config_file">config_file+ + @@ -398,6 +419,23 @@ +-# File lib/common/browser.rb, line 68 +# File lib/common/browser.rb, line 70 def load_config(config_file = nil) @config_file = config_file || @config_file if File.symlink?(@config_file) - raise "[ERROR] Config file is a symlink." + raise '[ERROR] Config file is a symlink.' else data = JSON.parse(File.read(@config_file)) end @@ -593,7 +593,7 @@ href="Browser.html#attribute-i-config_file">config_file+-# File lib/common/browser.rb, line 97 +# File lib/common/browser.rb, line 99 def merge_request_params(params = {}) params = Browser.append_params_header_field( params, @@ -617,6 +617,14 @@ href="Browser.html#attribute-i-config_file">config_file ) end + if @request_timeout + params = params.merge(timeout: @request_timeout) + end + + if @connect_timeout + params = params.merge(connecttimeout: @connect_timeout) + end + # Used to enable the cache system if :cache_ttl > 0 unless params.has_key?(:cache_ttl) params = params.merge(cache_ttl: @cache_ttl) diff --git a/doc_rdoc/Browser/Options.html b/doc_rdoc/Browser/Options.html index f7cc60fe..2a00cd02 100644 --- a/doc_rdoc/Browser/Options.html +++ b/doc_rdoc/Browser/Options.html @@ -59,6 +59,8 @@#basic_auth= +#connect_timeout= +#invalid_proxy_auth_format #max_threads @@ -71,6 +73,8 @@#proxy_auth= +#request_timeout= +#user_agent #user_agent_mode= @@ -368,6 +372,23 @@ +
Sets the connect timeout @param [ Integer ] timeout Timeout in ms
+ +@return [ void ]
+ + + ++# File lib/common/browser/options.rb, line 129 +def connect_timeout=(timeout) + @connect_timeout = timeout +end+
Sets the request timeout @param [ Integer ] timeout Timeout in ms
+ +@return [ void ]
+ + + ++# File lib/common/browser/options.rb, line 121 +def request_timeout=(timeout) + @request_timeout = timeout +end+
-# File lib/common/browser/options.rb, line 119 +# File lib/common/browser/options.rb, line 135 def invalid_proxy_auth_format 'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}' end@@ -810,7 +918,7 @@ href="Options.html#attribute-i-available_user_agents">available_user_agents<
-# File lib/common/browser/options.rb, line 127 +# File lib/common/browser/options.rb, line 143 def override_config(options = {}) options.each do |option, value| if value != nil and OPTIONS.include?(option) diff --git a/doc_rdoc/GenerateList.html b/doc_rdoc/GenerateList.html index 1a1b8782..bf5b214b 100644 --- a/doc_rdoc/GenerateList.html +++ b/doc_rdoc/GenerateList.html @@ -364,17 +364,17 @@ @type = 'plugin' @svn_url = 'http://plugins.svn.wordpress.org/' @popular_url = 'http://wordpress.org/plugins/browse/popular/' - @popular_regex = %{<h3><a href="http://wordpress.org/plugins/(.+)/">.+</a></h3>} + @popular_regex = %{<h3><a href="http://wordpress.org/plugins/([^/]+)/">.+</a></h3>} elsif type =~ /themes/ @type = 'theme' @svn_url = 'http://themes.svn.wordpress.org/' @popular_url = 'http://wordpress.org/themes/browse/popular/' - @popular_regex = %{<h3><a href="http://wordpress.org/themes/(.+)">.+</a></h3>} + @popular_regex = %{<h3><a href="http://wordpress.org/themes/([^/]+)">.+</a></h3>} else raise "Type #{type} not defined" end @verbose = verbose - @browser = Browser.instance + @browser = Browser.instance(request_timeout: 20000, connect_timeout: 20000, max_threads: 1) @hydra = @browser.hydra end
-# File lib/wpstools/plugins/list_generator/generate_list.rb, line 103 +# File lib/wpstools/plugins/list_generator/generate_list.rb, line 111 def save(items) items.sort! items.uniq! - puts "[*] We have parsed #{items.length} #@types" + puts "[*] We have parsed #{items.length} #{@type}s" File.open(@file_name, 'w') { |f| f.puts(items) } puts "New #@file_name file created" enddiff --git a/doc_rdoc/StatsPlugin.html b/doc_rdoc/StatsPlugin.html index ffe2b7fb..158762ba 100644 --- a/doc_rdoc/StatsPlugin.html +++ b/doc_rdoc/StatsPlugin.html @@ -416,7 +416,7 @@
# File lib/wpstools/plugins/stats/stats_plugin.rb, line 34 def plugin_vulns_count(file=PLUGINS_VULNS_FILE) - xml(file).xpath("count(//vulnerability)").to_i + xml(file).xpath('count(//vulnerability)').to_i end
# File lib/wpstools/plugins/stats/stats_plugin.rb, line 38 def theme_vulns_count(file=THEMES_VULNS_FILE) - xml(file).xpath("count(//vulnerability)").to_i + xml(file).xpath('count(//vulnerability)').to_i end
# File lib/wpstools/plugins/stats/stats_plugin.rb, line 26 def vuln_plugin_count(file=PLUGINS_VULNS_FILE) - xml(file).xpath("count(//plugin)").to_i + xml(file).xpath('count(//plugin)').to_i end
# File lib/wpstools/plugins/stats/stats_plugin.rb, line 30 def vuln_theme_count(file=THEMES_VULNS_FILE) - xml(file).xpath("count(//theme)").to_i + xml(file).xpath('count(//theme)').to_i enddiff --git a/doc_rdoc/created.rid b/doc_rdoc/created.rid index 76d98d23..d686258e 100644 --- a/doc_rdoc/created.rid +++ b/doc_rdoc/created.rid @@ -1,9 +1,9 @@ -Fri, 19 Jul 2013 21:49:32 +0200 +Fri, 19 Jul 2013 23:10:11 +0200 ./CREDITS Mon, 01 Apr 2013 23:09:01 +0200 ./Gemfile Sat, 22 Jun 2013 21:00:02 +0200 ./lib/common/browser/actions.rb Fri, 19 Jul 2013 13:03:39 +0200 -./lib/common/browser/options.rb Sun, 14 Apr 2013 10:46:08 +0200 -./lib/common/browser.rb Sun, 14 Apr 2013 10:46:08 +0200 +./lib/common/browser/options.rb Fri, 19 Jul 2013 22:41:06 +0200 +./lib/common/browser.rb Fri, 19 Jul 2013 22:41:36 +0200 ./lib/common/cache_file_store.rb Fri, 19 Jul 2013 10:55:20 +0200 ./lib/common/collections/vulnerabilities/output.rb Fri, 05 Apr 2013 20:07:17 +0200 ./lib/common/collections/vulnerabilities.rb Fri, 05 Apr 2013 20:07:17 +0200 @@ -71,10 +71,10 @@ Fri, 19 Jul 2013 21:49:32 +0200 ./lib/wpscan/wpscan_helper.rb Sun, 30 Jun 2013 13:29:48 +0200 ./lib/wpscan/wpscan_options.rb Sun, 09 Jun 2013 09:14:41 +0200 ./lib/wpstools/plugins/checker/checker_plugin.rb Fri, 05 Apr 2013 20:07:17 +0200 -./lib/wpstools/plugins/list_generator/generate_list.rb Fri, 14 Jun 2013 13:42:57 +0200 +./lib/wpstools/plugins/list_generator/generate_list.rb Fri, 19 Jul 2013 22:53:18 +0200 ./lib/wpstools/plugins/list_generator/list_generator_plugin.rb Fri, 05 Apr 2013 20:07:17 +0200 ./lib/wpstools/plugins/list_generator/svn_parser.rb Wed, 17 Apr 2013 21:56:58 +0200 -./lib/wpstools/plugins/stats/stats_plugin.rb Fri, 05 Apr 2013 20:07:17 +0200 +./lib/wpstools/plugins/stats/stats_plugin.rb Fri, 19 Jul 2013 22:13:30 +0200 ./lib/wpstools/wpstools_helper.rb Fri, 05 Apr 2013 20:07:17 +0200 ./LICENSE Fri, 05 Apr 2013 20:07:17 +0200 ./README Fri, 19 Jul 2013 10:55:20 +0200 diff --git a/doc_rdoc/index.html b/doc_rdoc/index.html index 84389e23..3b8af72d 100644 --- a/doc_rdoc/index.html +++ b/doc_rdoc/index.html @@ -249,37 +249,37 @@
#available_user_agents, #basic_auth, #cache_ttl, #proxy, #proxy_auth, #user_agent, #user_agent_mode
+#available_user_agents, #basic_auth, #cache_ttl, #connect_timeout, #proxy, #proxy_auth, #request_timeout, #user_agent, #user_agent_mode
@@ -499,8 +501,6 @@-29 -30 31 32 33 @@ -514,16 +514,18 @@ 41 42 43 -44+44 +45 +46
# File 'lib/common/browser.rb', line 29 +# File 'lib/common/browser.rb', line 31 def initialize( = {}) @config_file = [:config_file] || CONF_DIR + '/browser.conf.json' @cache_dir = [:cache_dir] || CACHE_DIR + '/browser' - load_config() + load_config override_config() unless @hydra @@ -573,12 +575,12 @@-24 -25 -26+26 +27 +28
# File 'lib/common/browser.rb', line 24 +# File 'lib/common/browser.rb', line 26 def cache_dir @cache_dir @@ -616,12 +618,12 @@-24 -25 -26+26 +27 +28
# File 'lib/common/browser.rb', line 24 +# File 'lib/common/browser.rb', line 26 def config_file @config_file @@ -659,12 +661,12 @@-24 -25 -26+26 +27 +28
# File 'lib/common/browser.rb', line 24 +# File 'lib/common/browser.rb', line 26 def hydra @hydra @@ -757,17 +759,17 @@-143 -144 -145 -146 -147 -148 -149 -150+153 +154 +155 +156 +157 +158 +159 +160
# File 'lib/common/browser.rb', line 143 +# File 'lib/common/browser.rb', line 153 def self.append_params_header_field(params = {}, field, field_value) if !params.has_key?(:headers) @@ -836,15 +838,15 @@-51 -52 53 54 55 -56+56 +57 +58
# File 'lib/common/browser.rb', line 51 +# File 'lib/common/browser.rb', line 53 def self.instance( = {}) unless @@instance @@ -872,12 +874,12 @@-58 -59 -60+60 +61 +62
# File 'lib/common/browser.rb', line 58 +# File 'lib/common/browser.rb', line 60 def self.reset @@instance = nil @@ -958,12 +960,12 @@-91 -92 -93+93 +94 +95
# File 'lib/common/browser.rb', line 91 +# File 'lib/common/browser.rb', line 93 def forge_request(url, params = {}) Typhoeus::Request.new(url, merge_request_params(params)) @@ -1016,8 +1018,6 @@-69 -70 71 72 73 @@ -1032,16 +1032,18 @@ 82 83 84 -85+85 +86 +87
# File 'lib/common/browser.rb', line 69 +# File 'lib/common/browser.rb', line 71 def load_config(config_file = nil) @config_file = config_file || @config_file if File.symlink?(@config_file) - raise "[ERROR] Config file is a symlink." + raise '[ERROR] Config file is a symlink.' else data = JSON.parse(File.read(@config_file)) end @@ -1113,8 +1115,6 @@-98 -99 100 101 102 @@ -1149,10 +1149,20 @@ 131 132 133 -134+134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144
# File 'lib/common/browser.rb', line 98 +# File 'lib/common/browser.rb', line 100 def merge_request_params(params = {}) params = Browser.append_params_header_field( @@ -1177,6 +1187,14 @@ ) end + if @request_timeout + params = params.merge(timeout: @request_timeout) + end + + if @connect_timeout + params = params.merge(connecttimeout: @connect_timeout) + end + # Used to enable the cache system if :cache_ttl > 0 unless params.has_key?(:cache_ttl) params = params.merge(cache_ttl: @cache_ttl) @@ -1201,7 +1219,7 @@ diff --git a/doc_yard/Browser/Actions.html b/doc_yard/Browser/Actions.html index f2241383..040f3f6f 100644 --- a/doc_yard/Browser/Actions.html +++ b/doc_yard/Browser/Actions.html @@ -629,7 +629,7 @@ diff --git a/doc_yard/Browser/Options.html b/doc_yard/Browser/Options.html index e6761808..16e2d1e9 100644 --- a/doc_yard/Browser/Options.html +++ b/doc_yard/Browser/Options.html @@ -185,6 +185,33 @@Returns the value of attribute cache_ttl.
+ + + +
Returns the value of attribute connect_timeout.
+Returns the value of attribute proxy_auth.
+ + + +Returns the value of attribute request_timeout.
+Returns the value of attribute connect_timeout
+ + +
+ + + +8 +9 +10+ |
+
+ # File 'lib/common/browser/options.rb', line 8 + +def connect_timeout + @connect_timeout +end+ |
+
Returns the value of attribute request_timeout
+ + +
+ + + +8 +9 +10+ |
+
+ # File 'lib/common/browser/options.rb', line 8 + +def request_timeout + @request_timeout +end+ |
+
-120 -121 -122+136 +137 +138
# File 'lib/common/browser/options.rb', line 120 +# File 'lib/common/browser/options.rb', line 136 def invalid_proxy_auth_format 'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}' @@ -913,16 +1053,16 @@-128 -129 -130 -131 -132 -133 -134+144 +145 +146 +147 +148 +149 +150
# File 'lib/common/browser/options.rb', line 128 +# File 'lib/common/browser/options.rb', line 144 def override_config( = {}) .each do |option, value| @@ -941,7 +1081,7 @@ diff --git a/doc_yard/CacheFileStore.html b/doc_yard/CacheFileStore.html index 9682568c..23606d5d 100644 --- a/doc_yard/CacheFileStore.html +++ b/doc_yard/CacheFileStore.html @@ -612,7 +612,7 @@ Marshal does not need any "require" diff --git a/doc_yard/CheckerPlugin.html b/doc_yard/CheckerPlugin.html index ed1f5967..63c5c7ab 100644 --- a/doc_yard/CheckerPlugin.html +++ b/doc_yard/CheckerPlugin.html @@ -585,7 +585,7 @@ diff --git a/doc_yard/CustomOptionParser.html b/doc_yard/CustomOptionParser.html index 580b3133..31013d79 100644 --- a/doc_yard/CustomOptionParser.html +++ b/doc_yard/CustomOptionParser.html @@ -643,7 +643,7 @@ diff --git a/doc_yard/File.html b/doc_yard/File.html index d67aba0c..dc9caf89 100644 --- a/doc_yard/File.html +++ b/doc_yard/File.html @@ -226,7 +226,7 @@ diff --git a/doc_yard/GenerateList.html b/doc_yard/GenerateList.html index 26c1feb7..2f166df2 100644 --- a/doc_yard/GenerateList.html +++ b/doc_yard/GenerateList.html @@ -357,17 +357,17 @@ parse the response for the names. @type = 'plugin' @svn_url = 'http://plugins.svn.wordpress.org/' @popular_url = 'http://wordpress.org/plugins/browse/popular/' - @popular_regex = %r{<h3><a href="http://wordpress.org/plugins/(.+)/">.+</a></h3>}i + @popular_regex = %r{<h3><a href="http://wordpress.org/plugins/([^/]+)/">.+</a></h3>}i elsif type =~ /themes/i @type = 'theme' @svn_url = 'http://themes.svn.wordpress.org/' @popular_url = 'http://wordpress.org/themes/browse/popular/' - @popular_regex = %r{<h3><a href="http://wordpress.org/themes/(.+)">.+</a></h3>}i + @popular_regex = %r{<h3><a href="http://wordpress.org/themes/([^/]+)">.+</a></h3>}i else raise "Type #{type} not defined" end @verbose = verbose - @browser = Browser.instance + @browser = Browser.instance(request_timeout: 20000, connect_timeout: 20000, max_threads: 1) @hydra = @browser.hydra end
# File 'lib/wpstools/plugins/list_generator/generate_list.rb', line 67 @@ -577,12 +585,20 @@ parse the response for the names. queue_count += 1 request.on_complete do |response| + if response.code != 200 + puts red("Got HTTP Status #{response.code} for page #{page}. Retrying request...") + # Retry + @hydra.queue(request) + next + end puts "[+] Parsing page #{page_count}" if @verbose page_count += 1 + found = 0 response.body.scan(@popular_regex).each do |item| - puts "[+] Found popular #@type: #{item}" if @verbose found_items << item[0] + found = found + 1 end + puts "[+] Found #{found} items on page #{page}" if @verbose end @hydra.queue(request) @@ -630,21 +646,21 @@ parse the response for the names.-104 -105 -106 -107 -108 -109 -110+112 +113 +114 +115 +116 +117 +118
# File 'lib/wpstools/plugins/list_generator/generate_list.rb', line 104 +# File 'lib/wpstools/plugins/list_generator/generate_list.rb', line 112 def save(items) items.sort! items.uniq! - puts "[*] We have parsed #{items.length} #@types" + puts "[*] We have parsed #{items.length} #{@type}s" File.open(@file_name, 'w') { |f| f.puts(items) } puts "New #@file_name file created" end@@ -730,7 +746,7 @@ parse the response for the names. diff --git a/doc_yard/GitUpdater.html b/doc_yard/GitUpdater.html index c26339d8..8e0a4af9 100644 --- a/doc_yard/GitUpdater.html +++ b/doc_yard/GitUpdater.html @@ -529,7 +529,7 @@ the last commit hash diff --git a/doc_yard/ListGeneratorPlugin.html b/doc_yard/ListGeneratorPlugin.html index 6506234a..5a340883 100644 --- a/doc_yard/ListGeneratorPlugin.html +++ b/doc_yard/ListGeneratorPlugin.html @@ -435,7 +435,7 @@ diff --git a/doc_yard/Plugin.html b/doc_yard/Plugin.html index 7d39f089..f0288e7b 100644 --- a/doc_yard/Plugin.html +++ b/doc_yard/Plugin.html @@ -506,7 +506,7 @@ diff --git a/doc_yard/Plugins.html b/doc_yard/Plugins.html index 6db1895e..55523a5a 100644 --- a/doc_yard/Plugins.html +++ b/doc_yard/Plugins.html @@ -462,7 +462,7 @@ diff --git a/doc_yard/StatsPlugin.html b/doc_yard/StatsPlugin.html index 8a5ccfe5..d57c7513 100644 --- a/doc_yard/StatsPlugin.html +++ b/doc_yard/StatsPlugin.html @@ -449,7 +449,7 @@# File 'lib/wpstools/plugins/stats/stats_plugin.rb', line 35 def plugin_vulns_count(file=PLUGINS_VULNS_FILE) - xml(file).xpath("count(//vulnerability)").to_i + xml(file).xpath('count(//vulnerability)').to_i end
# File 'lib/wpstools/plugins/stats/stats_plugin.rb', line 39 def theme_vulns_count(file=THEMES_VULNS_FILE) - xml(file).xpath("count(//vulnerability)").to_i + xml(file).xpath('count(//vulnerability)').to_i end@@ -619,7 +619,7 @@
# File 'lib/wpstools/plugins/stats/stats_plugin.rb', line 27 def vuln_plugin_count(file=PLUGINS_VULNS_FILE) - xml(file).xpath("count(//plugin)").to_i + xml(file).xpath('count(//plugin)').to_i end@@ -649,7 +649,7 @@
# File 'lib/wpstools/plugins/stats/stats_plugin.rb', line 31 def vuln_theme_count(file=THEMES_VULNS_FILE) - xml(file).xpath("count(//theme)").to_i + xml(file).xpath('count(//theme)').to_i end@@ -661,7 +661,7 @@