diff --git a/conf/browser.conf.json b/conf/browser.conf.json index 21a3821d..4c0e7a67 100644 --- a/conf/browser.conf.json +++ b/conf/browser.conf.json @@ -18,6 +18,8 @@ "request_timeout": 2000, // 2s + "connect_timeout": 1000, // 1s + "max_threads": 20, // Some user_agents can be found there http://techpatterns.com/downloads/firefox/useragentswitcher.xml (thx to Gianluca Brindisi) diff --git a/lib/common/browser.rb b/lib/common/browser.rb index 63017212..ce8ab062 100644 --- a/lib/common/browser.rb +++ b/lib/common/browser.rb @@ -16,7 +16,9 @@ class Browser :user_agent, :user_agent_mode, :proxy, - :proxy_auth + :proxy_auth, + :request_timeout, + :connect_timeout ] @@instance = nil @@ -30,7 +32,7 @@ class Browser @config_file = options[:config_file] || CONF_DIR + '/browser.conf.json' @cache_dir = options[:cache_dir] || CACHE_DIR + '/browser' - load_config() + load_config override_config(options) unless @hydra @@ -70,7 +72,7 @@ class Browser @config_file = config_file || @config_file if File.symlink?(@config_file) - raise "[ERROR] Config file is a symlink." + raise '[ERROR] Config file is a symlink.' else data = JSON.parse(File.read(@config_file)) end @@ -118,6 +120,14 @@ class Browser ) end + if @request_timeout + params = params.merge(timeout: @request_timeout) + end + + if @connect_timeout + params = params.merge(connecttimeout: @connect_timeout) + end + # Used to enable the cache system if :cache_ttl > 0 unless params.has_key?(:cache_ttl) params = params.merge(cache_ttl: @cache_ttl) diff --git a/lib/common/browser/options.rb b/lib/common/browser/options.rb index 3c5a47b2..750b8029 100644 --- a/lib/common/browser/options.rb +++ b/lib/common/browser/options.rb @@ -5,7 +5,7 @@ class Browser USER_AGENT_MODES = %w{ static semi-static random } - attr_accessor :available_user_agents, :cache_ttl + attr_accessor :available_user_agents, :cache_ttl, :request_timeout, :connect_timeout attr_reader :basic_auth, :user_agent_mode, :proxy, :proxy_auth attr_writer :user_agent @@ -115,6 +115,22 @@ class Browser end end + # Sets the request timeout + # @param [ Integer ] timeout Timeout in ms + # + # @return [ void ] + def request_timeout=(timeout) + @request_timeout = timeout + end + + # Sets the connect timeout + # @param [ Integer ] timeout Timeout in ms + # + # @return [ void ] + def connect_timeout=(timeout) + @connect_timeout = timeout + end + protected def invalid_proxy_auth_format diff --git a/lib/wpstools/plugins/list_generator/generate_list.rb b/lib/wpstools/plugins/list_generator/generate_list.rb index fe598066..2e77d724 100644 --- a/lib/wpstools/plugins/list_generator/generate_list.rb +++ b/lib/wpstools/plugins/list_generator/generate_list.rb @@ -11,17 +11,17 @@ class GenerateList @type = 'plugin' @svn_url = 'http://plugins.svn.wordpress.org/' @popular_url = 'http://wordpress.org/plugins/browse/popular/' - @popular_regex = %r{