Gemfile conflict

lib/common/browser.rb

@@ -1,42 +1,43 @@
 # encoding: UTF-8
 
 require 'common/typhoeus_cache'
+require 'common/browser/actions'
+require 'common/browser/options'
 
 class Browser
-  @@instance = nil
-  USER_AGENT_MODES = %w{ static semi-static random }
+  extend  Browser::Actions
+  include Browser::Options
 
-  ACCESSOR_OPTIONS = [
+  OPTIONS = [
+    :available_user_agents,
+    :basic_auth,
+    :cache_ttl,
+    :max_threads,
     :user_agent,
     :user_agent_mode,
-    :available_user_agents,
     :proxy,
-    :proxy_auth,
-    :max_threads,
-    :cache_ttl,
-    :request_timeout,
-    :basic_auth
+    :proxy_auth
   ]
 
-  attr_reader :hydra, :config_file
-  attr_accessor *ACCESSOR_OPTIONS
+  @@instance = nil
 
+  attr_reader :hydra, :config_file, :cache_dir
+
+  # @param [ Hash ] options
+  #
+  # @return [ Browser ]
   def initialize(options = {})
     @config_file = options[:config_file] || CONF_DIR + '/browser.conf.json'
-    options.delete(:config_file)
+    @cache_dir   = options[:cache_dir]   || CACHE_DIR + '/browser'
 
     load_config()
+    override_config(options)
 
-    if options.length > 0
-      override_config_with_options(options)
+    unless @hydra
+      @hydra = Typhoeus::Hydra.new(max_concurrency: self.max_threads)
     end
 
-    @hydra = Typhoeus::Hydra.new(max_concurrency: @max_threads)
-
-    # TODO : add an argument for the cache dir instead of using a constant
-    @cache_dir = CACHE_DIR + '/browser'
     @cache = TyphoeusCache.new(@cache_dir)
-
     @cache.clean
 
     Typhoeus::Config.cache = @cache
@@ -44,6 +45,9 @@ class Browser
 
   private_class_method :new
 
+  # @param [ Hash ] options
+  #
+  # @return [ Browser ]
   def self.instance(options = {})
     unless @@instance
       @@instance = new(options)
@@ -55,57 +59,13 @@ class Browser
     @@instance = nil
   end
 
-  def user_agent_mode=(ua_mode)
-    ua_mode ||= 'static'
-
-    if USER_AGENT_MODES.include?(ua_mode)
-      @user_agent_mode = ua_mode
-      # For semi-static user agent mode, the user agent has to
-      # be nil the first time (it will be set with the getter)
-      @user_agent = nil if ua_mode === 'semi-static'
-    else
-      raise "Unknow user agent mode : '#{ua_mode}'"
-    end
-  end
-
-  # return the user agent, according to the user_agent_mode
-  def user_agent
-    case @user_agent_mode
-    when 'semi-static'
-      unless @user_agent
-        @user_agent = @available_user_agents.sample
-      end
-    when 'random'
-      @user_agent = @available_user_agents.sample
-    end
-    @user_agent
-  end
-
-  def max_threads=(max_threads)
-    if max_threads.nil? or max_threads <= 0
-      max_threads = 1
-    end
-    @max_threads = max_threads
-  end
-
-  def proxy_auth=(auth)
-    unless auth.nil?
-      if auth.is_a?(Hash) && auth.include?(:proxy_username) && auth.include?(:proxy_password)
-        @proxy_auth = auth[:proxy_username] + ':' + auth[:proxy_password]
-      elsif auth.is_a?(String) && auth.index(':') != nil
-        @proxy_auth = auth
-      else
-        raise invalid_proxy_auth_format
-      end
-    end
-  end
-
-  def invalid_proxy_auth_format
-    'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}'
-  end
-
-  # TODO reload hydra (if the .load_config is called on a browser object,
-  # hydra will not have the new @max_threads and @request_timeout)
+  #
+  # If an option was set but is not in the new config_file
+  # it's value is kept
+  #
+  # @param [ String ] config_file
+  #
+  # @return [ void ]
   def load_config(config_file = nil)
     @config_file = config_file || @config_file
 
@@ -115,40 +75,26 @@ class Browser
       data = JSON.parse(File.read(@config_file))
     end
 
-    ACCESSOR_OPTIONS.each do |option|
+    OPTIONS.each do |option|
       option_name = option.to_s
 
-      self.send(:"#{option_name}=", data[option_name])
+      unless data[option_name].nil?
+        self.send(:"#{option_name}=", data[option_name])
+      end
     end
   end
 
-  def get(url, params = {})
-    run_request(
-      forge_request(url, params.merge(method: :get))
-    )
-  end
-
-  def post(url, params = {})
-    run_request(
-      forge_request(url, params.merge(method: :post))
-    )
-  end
-
-  def get_and_follow_location(url, params = {})
-    params[:maxredirs] ||= 2
-
-    run_request(
-      forge_request(url, params.merge(method: :get, followlocation: true))
-    )
-  end
-
+  # @param [ String ] url
+  # @param [ Hash ] params
+  #
+  # @return [ Typhoeus::Request ]
   def forge_request(url, params = {})
-    Typhoeus::Request.new(
-      url.to_s,
-      merge_request_params(params)
-    )
+    Typhoeus::Request.new(url, merge_request_params(params))
   end
 
+  # @param [ Hash ] params
+  #
+  # @return [ Hash ]
   def merge_request_params(params = {})
     params = Browser.append_params_header_field(
       params,
@@ -189,7 +135,11 @@ class Browser
 
   private
 
-  # return Array
+  # @param [ Hash ] params
+  # @param [ String ] field
+  # @param [ Mixed ] field_value
+  #
+  # @return [ Array ]
   def self.append_params_header_field(params = {}, field, field_value)
     if !params.has_key?(:headers)
       params = params.merge(:headers => { field => field_value })
@@ -199,19 +149,4 @@ class Browser
     params
   end
 
-  # return the response
-  def run_request(request)
-    @hydra.queue request
-    @hydra.run
-    request.response
-  end
-
-  # Override with the options if they are set
-  def override_config_with_options(options)
-    options.each do |option, value|
-      if value != nil and ACCESSOR_OPTIONS.include?(option)
-        self.send(:"#{option}=", value)
-      end
-    end
-  end
 end

lib/common/browser/actions.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+
+class Browser
+  module Actions
+
+    # @param [ String ] url
+    # @param [ Hash ] params
+    #
+    # @return [ Typhoeus::Response ]
+    def get(url, params = {})
+      process(url, params.merge(method: :get))
+    end
+
+    # @param [ String ] url
+    # @param [ Hash ] params
+    #
+    # @return [ Typhoeus::Response ]
+    def post(url, params = {})
+      process(url, params.merge(method: :post))
+    end
+
+    # @param [ String ] url
+    # @param [ Hash ] params
+    #
+    # @return [ Typhoeus::Response ]
+    def get_and_follow_location(url, params = {})
+      params[:maxredirs] ||= 2
+
+      get(url, params.merge(followlocation: true))
+    end
+
+    protected
+
+    # @param [ String ] url
+    # @param [ Hash ] params
+    #
+    # @return [ Typhoeus::Response ]
+    def process(url, params)
+      Typhoeus::Request.new(url, Browser.instance.merge_request_params(params)).run
+    end
+
+  end
+end

lib/common/browser/options.rb

@@ -0,0 +1,137 @@
+# encoding: UTF-8
+
+class Browser
+  module Options
+
+    USER_AGENT_MODES = %w{ static semi-static random }
+
+    attr_accessor :available_user_agents, :cache_ttl
+    attr_reader   :basic_auth, :user_agent_mode, :proxy, :proxy_auth
+    attr_writer   :user_agent
+
+    # Sets the Basic Authentification credentials
+    # Accepted format:
+    #   login:password
+    #   Basic base_64_encoded
+    #
+    # @param [ String ] auth
+    #
+    # @return [ void ]
+    def basic_auth=(auth)
+      if auth.index(':')
+        @basic_auth = "Basic #{Base64.encode64(auth).chomp}"
+      elsif auth =~ /\ABasic [a-zA-Z0-9=]+\z/
+        @basic_auth = auth
+      else
+       raise 'Invalid basic authentication format, "login:password" or "Basic base_64_encoded" expected'
+     end
+    end
+
+    # @return [ Integer ]
+    def max_threads
+      @max_threads || 1
+    end
+
+    def max_threads=(threads)
+      if threads.is_a?(Integer) && threads > 0
+        @max_threads = threads
+        @hydra = Typhoeus::Hydra.new(max_concurrency: threads)
+      else
+        raise 'max_threads must be an Integer > 0'
+      end
+    end
+
+    # Sets the user_agent_mode, which can be one of the following:
+    #   static:      The UA is defined by the user, and will be the same in each requests
+    #   semi-static: The UA is randomly chosen at the first request, and will not change
+    #   random:      UA randomly chosen each request
+    #
+    # UA are from @available_user_agents
+    #
+    # @param [ String ] ua_mode
+    #
+    # @return [ void ]
+    def user_agent_mode=(ua_mode)
+      ua_mode ||= 'static'
+
+      if USER_AGENT_MODES.include?(ua_mode)
+        @user_agent_mode = ua_mode
+        # For semi-static user agent mode, the user agent has to
+        # be nil the first time (it will be set with the getter)
+        @user_agent = nil if ua_mode === 'semi-static'
+      else
+        raise "Unknow user agent mode : '#{ua_mode}'"
+      end
+    end
+
+    # @return [ String ] The user agent, according to the user_agent_mode
+    def user_agent
+      case @user_agent_mode
+      when 'semi-static'
+        unless @user_agent
+          @user_agent = @available_user_agents.sample
+        end
+      when 'random'
+        @user_agent = @available_user_agents.sample
+      end
+      @user_agent
+    end
+
+    # Sets the proxy
+    # Accepted format:
+    #   [protocol://]host:post
+    #
+    #  Supported protocols:
+    #    Depends on the curl protocols, See curl --version
+    #
+    # @param [ String ] proxy
+    #
+    # @return [ void ]
+    def proxy=(proxy)
+      if proxy.index(':')
+        @proxy = proxy
+      else
+        raise 'Invalid proxy format. Should be [protocol://]host:port.'
+      end
+    end
+
+    # Sets the proxy credentials
+    # Accepted format:
+    #   username:password
+    #   { proxy_username: username, :proxy_password: password }
+    #
+    # @param [ String ] auth
+    #
+    # @return [ void ]
+    def proxy_auth=(auth)
+      unless auth.nil?
+        if auth.is_a?(Hash) && auth.include?(:proxy_username) && auth.include?(:proxy_password)
+          @proxy_auth = auth[:proxy_username] + ':' + auth[:proxy_password]
+        elsif auth.is_a?(String) && auth.index(':') != nil
+          @proxy_auth = auth
+        else
+          raise invalid_proxy_auth_format
+        end
+      end
+    end
+
+    protected
+
+    def invalid_proxy_auth_format
+      'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}'
+    end
+
+    # Override with the options if they are set
+    # @param [ Hash ] options
+    #
+    # @return [ void ]
+    def override_config(options = {})
+      options.each do |option, value|
+        if value != nil and OPTIONS.include?(option)
+          self.send(:"#{option}=", value)
+        end
+      end
+    end
+
+  end
+end

lib/common/collections/wp_items/detectable.rb

@@ -67,7 +67,7 @@ class WpItems < Array
       results      = new
       item_class   = self.item_class
       type         = self.to_s.gsub(/Wp/, '').downcase
-      response     = Browser.instance.get(wp_target.url)
+      response     = Browser.get(wp_target.url)
       item_options = {
         wp_content_dir: wp_target.wp_content_dir,
         wp_plugins_dir: wp_target.wp_plugins_dir,

lib/common/collections/wp_users/output.rb

@@ -10,7 +10,7 @@ class WpUsers < WpItems
       max_display_name_length = self.sort { |a, b| a.display_name.length <=> b.display_name.length }.last.display_name.length
 
       inner_space         = 2
-      id_length           = (max_id_length + inner_space * 2) /2 *2
+      id_length           = (max_id_length + inner_space * 2) /2 * 2
       login_length        = max_login_length + inner_space * 2
       display_name_length = max_display_name_length + inner_space * 2
 

lib/common/common_helper.rb

@@ -107,3 +107,8 @@ def xml(file)
     config.noblanks
   end
 end
+
+def redefine_constant(constant, value)
+  Object.send(:remove_const, constant)
+  Object.const_set(constant, value)
+end

lib/common/models/wp_item/existable.rb

@@ -13,7 +13,7 @@ class WpItem
     # @return [ Boolean ]
     def exists?(options = {}, response = nil)
       unless response
-        response = Browser.instance.get(url)
+        response = Browser.get(url)
       end
       exists_from_response?(response, options)
     end

lib/common/models/wp_item/infos.rb

@@ -7,7 +7,7 @@ class WpItem
 
     # @return [ Boolean ]
     def has_readme?
-      Browser.instance.get(readme_url).code == 200 ? true : false
+      Browser.get(readme_url).code == 200 ? true : false
     end
 
     # @return [ String ] The url to the readme file
@@ -17,7 +17,7 @@ class WpItem
 
     # @return [ Boolean ]
     def has_changelog?
-      Browser.instance.get(changelog_url).code == 200 ? true : false
+      Browser.get(changelog_url).code == 200 ? true : false
     end
 
     # @return [ String ] The url to the changelog file
@@ -27,7 +27,7 @@ class WpItem
 
     # @return [ Boolean ]
     def has_directory_listing?
-      Browser.instance.get(@uri.to_s).body[%r{<title>Index of}] ? true : false
+      Browser.get(@uri.to_s).body[%r{<title>Index of}] ? true : false
     end
 
     # Discover any error_log files created by WordPress
@@ -41,7 +41,7 @@ class WpItem
     #
     # @return [ Boolean ]
     def has_error_log?
-      response_body = Browser.instance.get(error_log_url, headers: {'range' => 'bytes=0-700'}).body
+      response_body = Browser.get(error_log_url, headers: {'range' => 'bytes=0-700'}).body
       response_body[%r{PHP Fatal error}i] ? true : false
     end
 

lib/common/models/wp_item/versionable.rb

@@ -10,7 +10,7 @@ class WpItem
     # @return [ String ] The version number
     def version
       unless @version
-        response = Browser.instance.get(readme_url)
+        response = Browser.get(readme_url)
         @version = response.body[%r{stable tag: #{WpVersion.version_pattern}}i, 1]
       end
       @version

lib/common/models/wp_theme/findable.rb

@@ -27,7 +27,7 @@ class WpTheme < WpItem
     #
     # @return [ WpTheme ]
     def find_from_css_link(target_uri)
-      response = Browser.instance.get_and_follow_location(target_uri.to_s)
+      response = Browser.get_and_follow_location(target_uri.to_s)
 
       # https + domain is optional because of relative links
       matches = %r{(?:https?://[^"']+)?/([^/]+)/themes/([^"']+)/style.css}i.match(response.body)
@@ -49,7 +49,7 @@ class WpTheme < WpItem
     #
     # @return [ WpTheme ]
     def find_from_wooframework(target_uri)
-      body = Browser.instance.get(target_uri.to_s).body
+      body = Browser.get(target_uri.to_s).body
       regexp = %r{<meta name="generator" content="([^\s"]+)\s?([^"]+)?" />\s+<meta name="generator" content="WooFramework\s?([^"]+)?" />}
 
 

lib/common/models/wp_theme/versionable.rb

@@ -5,7 +5,7 @@ class WpTheme < WpItem
 
     def version
       unless @version
-        @version = Browser.instance.get(style_url).body[%r{Version:\s([^\s]+)}i, 1]
+        @version = Browser.get(style_url).body[%r{Version:\s([^\s]+)}i, 1]
 
         # Get Version from readme.txt
         @version ||= super

lib/common/models/wp_timthumb/versionable.rb

@@ -9,7 +9,7 @@ class WpTimthumb < WpItem
     # @return [ String ] The version
     def version
       unless @version
-        response = Browser.instance.get(url)
+        response = Browser.get(url)
         @version = response.body[%r{TimThumb version\s*: ([^<]+)} , 1]
       end
       @version

lib/common/models/wp_user/existable.rb

@@ -24,7 +24,7 @@ class WpUser < WpItem
 
         @login        = Existable.login_from_author_pattern(location)
         @display_name = Existable.display_name_from_body(
-          Browser.instance.get(location).body
+          Browser.get(location).body
         )
       elsif response.code == 200 # login in body?
         @login        = Existable.login_from_body(response.body)

lib/common/models/wp_version/findable.rb

@@ -45,7 +45,7 @@ class WpVersion < WpItem
     # @return [ String ]
     def scan_url(target_uri, pattern, path = nil)
       url = path ? target_uri.merge(path).to_s : target_uri.to_s
-      response = Browser.instance.get_and_follow_location(url)
+      response = Browser.get_and_follow_location(url)
 
       response.body[pattern, 1]
     end
@@ -163,7 +163,7 @@ class WpVersion < WpItem
       xml.xpath('//file').each do |node|
         wp_item.path = node.attribute('src').text
 
-        response = Browser.instance.get(wp_item.url)
+        response = Browser.get(wp_item.url)
         md5sum = Digest::MD5.hexdigest(response.body)
 
         node.search('hash').each do |hash|