class Browser

Constants

ACCESSOR_OPTIONS
USER_AGENT_MODES

Attributes

config_file[R]
hydra[R]

Public Class Methods

instance(options = {}) click to toggle source
# File lib/browser.rb, line 64
def self.instance(options = {})
  unless @@instance
    @@instance = new(options)
  end
  @@instance
end
reset() click to toggle source
# File lib/browser.rb, line 71
def self.reset
  @@instance = nil
end

Public Instance Methods

forge_request(url, params = {}) click to toggle source
# File lib/browser.rb, line 178
def forge_request(url, params = {})
  Typhoeus::Request.new(
    url.to_s,
    merge_request_params(params)
  )
end
get(url, params = {}) click to toggle source
# File lib/browser.rb, line 166
def get(url, params = {})
  run_request(
    forge_request(url, params.merge(:method => :get))
  )
end
load_config(config_file = nil) click to toggle source

TODO reload hydra (if the .#load_config is called on a browser object, hydra will not have the new @max_threads and @request_timeout)

# File lib/browser.rb, line 136
def load_config(config_file = nil)
  @config_file = config_file || @config_file

  data = JSON.parse(File.read(@config_file))

  ACCESSOR_OPTIONS.each do |option|
    option_name = option.to_s

    self.send(:"#{option_name}=", data[option_name])
  end
end
max_threads=(max_threads) click to toggle source
# File lib/browser.rb, line 101
def max_threads=(max_threads)
  if max_threads.nil? or max_threads <= 0
    max_threads = 1
  end
  @max_threads = max_threads
end
merge_request_params(params = {}) click to toggle source
# File lib/browser.rb, line 185
def merge_request_params(params = {})
  if @proxy
    params = params.merge(:proxy => @proxy)

    if @proxy_auth
      params = params.merge(@proxy_auth)
    end
  end

  if @basic_auth
    if !params.has_key?(:headers)
      params = params.merge(:headers => {'Authorization' => @basic_auth})
    elsif !params[:headers].has_key?('Authorization')
      params[:headers]['Authorization'] = @basic_auth
    end
  end

  unless params.has_key?(:disable_ssl_host_verification)
    params = params.merge(:disable_ssl_host_verification => true)
  end

  unless params.has_key?(:disable_ssl_peer_verification)
    params = params.merge(:disable_ssl_peer_verification => true)
  end

  if !params.has_key?(:headers)
    params = params.merge(:headers => {'user-agent' => self.user_agent})
  elsif !params[:headers].has_key?('user-agent')
    params[:headers]['user-agent'] = self.user_agent
  end

  # Used to enable the cache system if :cache_timeout > 0
  unless params.has_key?(:cache_timeout)
    params = params.merge(:cache_timeout => @cache_timeout)
  end

  params
end
post(url, params = {}) click to toggle source
# File lib/browser.rb, line 172
def post(url, params = {})
  run_request(
    forge_request(url, params.merge(:method => :post))
  )
end
proxy_auth=(auth) click to toggle source
# File lib/browser.rb, line 108
def proxy_auth=(auth)
  unless auth.nil?
    if auth.is_a?(Hash)
      if !auth.include?(:proxy_username) or !auth.include?(:proxy_password)
        raise_invalid_proxy_format()
      end
      @proxy_auth = auth
    elsif auth.is_a?(String)
      if matches = %r{([^:]+):(.*)}.match(auth)
        @proxy_auth = {
          proxy_username: matches[1],
          proxy_password: matches[2]
        }
      else
        raise_invalid_proxy_auth_format()
      end
    else
      raise_invalid_proxy_auth_format()
    end
  end
end
raise_invalid_proxy_auth_format() click to toggle source
# File lib/browser.rb, line 130
def raise_invalid_proxy_auth_format
  raise 'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}'
end
user_agent() click to toggle source

return the user agent, according to the user_agent_mode

# File lib/browser.rb, line 89
def user_agent
  case @user_agent_mode
  when 'semi-static'
    unless @user_agent
      @user_agent = @available_user_agents.sample
    end
  when 'random'
    @user_agent = @available_user_agents.sample
  end
  @user_agent
end
user_agent_mode=(ua_mode) click to toggle source
# File lib/browser.rb, line 75
def user_agent_mode=(ua_mode)
  ua_mode ||= 'static'

  if USER_AGENT_MODES.include?(ua_mode)
    @user_agent_mode = ua_mode
    # For semi-static user agent mode, the user agent has to
    # be nil the first time (it will be set with the getter)
    @user_agent = nil if ua_mode === 'semi-static'
  else
    raise "Unknow user agent mode : '#{ua_mode}'"
  end
end