Gemfile conflict
This commit is contained in:
2
Gemfile
2
Gemfile
@@ -1,6 +1,6 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
# Seg fault in Typhoeus 0.6.3 (and ethon > 0.5.11 with rspec
|
||||
# Seg fault in Typhoeus 0.6.3 (and ethon > 0.5.11) with rspec
|
||||
gem "typhoeus", "=0.6.2"
|
||||
gem "ethon", "=0.5.10"
|
||||
gem "nokogiri"
|
||||
|
||||
@@ -1,42 +1,43 @@
|
||||
# encoding: UTF-8
|
||||
|
||||
require 'common/typhoeus_cache'
|
||||
require 'common/browser/actions'
|
||||
require 'common/browser/options'
|
||||
|
||||
class Browser
|
||||
@@instance = nil
|
||||
USER_AGENT_MODES = %w{ static semi-static random }
|
||||
extend Browser::Actions
|
||||
include Browser::Options
|
||||
|
||||
ACCESSOR_OPTIONS = [
|
||||
OPTIONS = [
|
||||
:available_user_agents,
|
||||
:basic_auth,
|
||||
:cache_ttl,
|
||||
:max_threads,
|
||||
:user_agent,
|
||||
:user_agent_mode,
|
||||
:available_user_agents,
|
||||
:proxy,
|
||||
:proxy_auth,
|
||||
:max_threads,
|
||||
:cache_ttl,
|
||||
:request_timeout,
|
||||
:basic_auth
|
||||
:proxy_auth
|
||||
]
|
||||
|
||||
attr_reader :hydra, :config_file
|
||||
attr_accessor *ACCESSOR_OPTIONS
|
||||
@@instance = nil
|
||||
|
||||
attr_reader :hydra, :config_file, :cache_dir
|
||||
|
||||
# @param [ Hash ] options
|
||||
#
|
||||
# @return [ Browser ]
|
||||
def initialize(options = {})
|
||||
@config_file = options[:config_file] || CONF_DIR + '/browser.conf.json'
|
||||
options.delete(:config_file)
|
||||
@cache_dir = options[:cache_dir] || CACHE_DIR + '/browser'
|
||||
|
||||
load_config()
|
||||
override_config(options)
|
||||
|
||||
if options.length > 0
|
||||
override_config_with_options(options)
|
||||
unless @hydra
|
||||
@hydra = Typhoeus::Hydra.new(max_concurrency: self.max_threads)
|
||||
end
|
||||
|
||||
@hydra = Typhoeus::Hydra.new(max_concurrency: @max_threads)
|
||||
|
||||
# TODO : add an argument for the cache dir instead of using a constant
|
||||
@cache_dir = CACHE_DIR + '/browser'
|
||||
@cache = TyphoeusCache.new(@cache_dir)
|
||||
|
||||
@cache.clean
|
||||
|
||||
Typhoeus::Config.cache = @cache
|
||||
@@ -44,6 +45,9 @@ class Browser
|
||||
|
||||
private_class_method :new
|
||||
|
||||
# @param [ Hash ] options
|
||||
#
|
||||
# @return [ Browser ]
|
||||
def self.instance(options = {})
|
||||
unless @@instance
|
||||
@@instance = new(options)
|
||||
@@ -55,57 +59,13 @@ class Browser
|
||||
@@instance = nil
|
||||
end
|
||||
|
||||
def user_agent_mode=(ua_mode)
|
||||
ua_mode ||= 'static'
|
||||
|
||||
if USER_AGENT_MODES.include?(ua_mode)
|
||||
@user_agent_mode = ua_mode
|
||||
# For semi-static user agent mode, the user agent has to
|
||||
# be nil the first time (it will be set with the getter)
|
||||
@user_agent = nil if ua_mode === 'semi-static'
|
||||
else
|
||||
raise "Unknow user agent mode : '#{ua_mode}'"
|
||||
end
|
||||
end
|
||||
|
||||
# return the user agent, according to the user_agent_mode
|
||||
def user_agent
|
||||
case @user_agent_mode
|
||||
when 'semi-static'
|
||||
unless @user_agent
|
||||
@user_agent = @available_user_agents.sample
|
||||
end
|
||||
when 'random'
|
||||
@user_agent = @available_user_agents.sample
|
||||
end
|
||||
@user_agent
|
||||
end
|
||||
|
||||
def max_threads=(max_threads)
|
||||
if max_threads.nil? or max_threads <= 0
|
||||
max_threads = 1
|
||||
end
|
||||
@max_threads = max_threads
|
||||
end
|
||||
|
||||
def proxy_auth=(auth)
|
||||
unless auth.nil?
|
||||
if auth.is_a?(Hash) && auth.include?(:proxy_username) && auth.include?(:proxy_password)
|
||||
@proxy_auth = auth[:proxy_username] + ':' + auth[:proxy_password]
|
||||
elsif auth.is_a?(String) && auth.index(':') != nil
|
||||
@proxy_auth = auth
|
||||
else
|
||||
raise invalid_proxy_auth_format
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def invalid_proxy_auth_format
|
||||
'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}'
|
||||
end
|
||||
|
||||
# TODO reload hydra (if the .load_config is called on a browser object,
|
||||
# hydra will not have the new @max_threads and @request_timeout)
|
||||
#
|
||||
# If an option was set but is not in the new config_file
|
||||
# it's value is kept
|
||||
#
|
||||
# @param [ String ] config_file
|
||||
#
|
||||
# @return [ void ]
|
||||
def load_config(config_file = nil)
|
||||
@config_file = config_file || @config_file
|
||||
|
||||
@@ -115,40 +75,26 @@ class Browser
|
||||
data = JSON.parse(File.read(@config_file))
|
||||
end
|
||||
|
||||
ACCESSOR_OPTIONS.each do |option|
|
||||
OPTIONS.each do |option|
|
||||
option_name = option.to_s
|
||||
|
||||
self.send(:"#{option_name}=", data[option_name])
|
||||
unless data[option_name].nil?
|
||||
self.send(:"#{option_name}=", data[option_name])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def get(url, params = {})
|
||||
run_request(
|
||||
forge_request(url, params.merge(method: :get))
|
||||
)
|
||||
end
|
||||
|
||||
def post(url, params = {})
|
||||
run_request(
|
||||
forge_request(url, params.merge(method: :post))
|
||||
)
|
||||
end
|
||||
|
||||
def get_and_follow_location(url, params = {})
|
||||
params[:maxredirs] ||= 2
|
||||
|
||||
run_request(
|
||||
forge_request(url, params.merge(method: :get, followlocation: true))
|
||||
)
|
||||
end
|
||||
|
||||
# @param [ String ] url
|
||||
# @param [ Hash ] params
|
||||
#
|
||||
# @return [ Typhoeus::Request ]
|
||||
def forge_request(url, params = {})
|
||||
Typhoeus::Request.new(
|
||||
url.to_s,
|
||||
merge_request_params(params)
|
||||
)
|
||||
Typhoeus::Request.new(url, merge_request_params(params))
|
||||
end
|
||||
|
||||
# @param [ Hash ] params
|
||||
#
|
||||
# @return [ Hash ]
|
||||
def merge_request_params(params = {})
|
||||
params = Browser.append_params_header_field(
|
||||
params,
|
||||
@@ -189,7 +135,11 @@ class Browser
|
||||
|
||||
private
|
||||
|
||||
# return Array
|
||||
# @param [ Hash ] params
|
||||
# @param [ String ] field
|
||||
# @param [ Mixed ] field_value
|
||||
#
|
||||
# @return [ Array ]
|
||||
def self.append_params_header_field(params = {}, field, field_value)
|
||||
if !params.has_key?(:headers)
|
||||
params = params.merge(:headers => { field => field_value })
|
||||
@@ -199,19 +149,4 @@ class Browser
|
||||
params
|
||||
end
|
||||
|
||||
# return the response
|
||||
def run_request(request)
|
||||
@hydra.queue request
|
||||
@hydra.run
|
||||
request.response
|
||||
end
|
||||
|
||||
# Override with the options if they are set
|
||||
def override_config_with_options(options)
|
||||
options.each do |option, value|
|
||||
if value != nil and ACCESSOR_OPTIONS.include?(option)
|
||||
self.send(:"#{option}=", value)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
43
lib/common/browser/actions.rb
Normal file
43
lib/common/browser/actions.rb
Normal file
@@ -0,0 +1,43 @@
|
||||
# encoding: UTF-8
|
||||
|
||||
class Browser
|
||||
module Actions
|
||||
|
||||
# @param [ String ] url
|
||||
# @param [ Hash ] params
|
||||
#
|
||||
# @return [ Typhoeus::Response ]
|
||||
def get(url, params = {})
|
||||
process(url, params.merge(method: :get))
|
||||
end
|
||||
|
||||
# @param [ String ] url
|
||||
# @param [ Hash ] params
|
||||
#
|
||||
# @return [ Typhoeus::Response ]
|
||||
def post(url, params = {})
|
||||
process(url, params.merge(method: :post))
|
||||
end
|
||||
|
||||
# @param [ String ] url
|
||||
# @param [ Hash ] params
|
||||
#
|
||||
# @return [ Typhoeus::Response ]
|
||||
def get_and_follow_location(url, params = {})
|
||||
params[:maxredirs] ||= 2
|
||||
|
||||
get(url, params.merge(followlocation: true))
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
# @param [ String ] url
|
||||
# @param [ Hash ] params
|
||||
#
|
||||
# @return [ Typhoeus::Response ]
|
||||
def process(url, params)
|
||||
Typhoeus::Request.new(url, Browser.instance.merge_request_params(params)).run
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
137
lib/common/browser/options.rb
Normal file
137
lib/common/browser/options.rb
Normal file
@@ -0,0 +1,137 @@
|
||||
# encoding: UTF-8
|
||||
|
||||
class Browser
|
||||
module Options
|
||||
|
||||
USER_AGENT_MODES = %w{ static semi-static random }
|
||||
|
||||
attr_accessor :available_user_agents, :cache_ttl
|
||||
attr_reader :basic_auth, :user_agent_mode, :proxy, :proxy_auth
|
||||
attr_writer :user_agent
|
||||
|
||||
# Sets the Basic Authentification credentials
|
||||
# Accepted format:
|
||||
# login:password
|
||||
# Basic base_64_encoded
|
||||
#
|
||||
# @param [ String ] auth
|
||||
#
|
||||
# @return [ void ]
|
||||
def basic_auth=(auth)
|
||||
if auth.index(':')
|
||||
@basic_auth = "Basic #{Base64.encode64(auth).chomp}"
|
||||
elsif auth =~ /\ABasic [a-zA-Z0-9=]+\z/
|
||||
@basic_auth = auth
|
||||
else
|
||||
raise 'Invalid basic authentication format, "login:password" or "Basic base_64_encoded" expected'
|
||||
end
|
||||
end
|
||||
|
||||
# @return [ Integer ]
|
||||
def max_threads
|
||||
@max_threads || 1
|
||||
end
|
||||
|
||||
def max_threads=(threads)
|
||||
if threads.is_a?(Integer) && threads > 0
|
||||
@max_threads = threads
|
||||
@hydra = Typhoeus::Hydra.new(max_concurrency: threads)
|
||||
else
|
||||
raise 'max_threads must be an Integer > 0'
|
||||
end
|
||||
end
|
||||
|
||||
# Sets the user_agent_mode, which can be one of the following:
|
||||
# static: The UA is defined by the user, and will be the same in each requests
|
||||
# semi-static: The UA is randomly chosen at the first request, and will not change
|
||||
# random: UA randomly chosen each request
|
||||
#
|
||||
# UA are from @available_user_agents
|
||||
#
|
||||
# @param [ String ] ua_mode
|
||||
#
|
||||
# @return [ void ]
|
||||
def user_agent_mode=(ua_mode)
|
||||
ua_mode ||= 'static'
|
||||
|
||||
if USER_AGENT_MODES.include?(ua_mode)
|
||||
@user_agent_mode = ua_mode
|
||||
# For semi-static user agent mode, the user agent has to
|
||||
# be nil the first time (it will be set with the getter)
|
||||
@user_agent = nil if ua_mode === 'semi-static'
|
||||
else
|
||||
raise "Unknow user agent mode : '#{ua_mode}'"
|
||||
end
|
||||
end
|
||||
|
||||
# @return [ String ] The user agent, according to the user_agent_mode
|
||||
def user_agent
|
||||
case @user_agent_mode
|
||||
when 'semi-static'
|
||||
unless @user_agent
|
||||
@user_agent = @available_user_agents.sample
|
||||
end
|
||||
when 'random'
|
||||
@user_agent = @available_user_agents.sample
|
||||
end
|
||||
@user_agent
|
||||
end
|
||||
|
||||
# Sets the proxy
|
||||
# Accepted format:
|
||||
# [protocol://]host:post
|
||||
#
|
||||
# Supported protocols:
|
||||
# Depends on the curl protocols, See curl --version
|
||||
#
|
||||
# @param [ String ] proxy
|
||||
#
|
||||
# @return [ void ]
|
||||
def proxy=(proxy)
|
||||
if proxy.index(':')
|
||||
@proxy = proxy
|
||||
else
|
||||
raise 'Invalid proxy format. Should be [protocol://]host:port.'
|
||||
end
|
||||
end
|
||||
|
||||
# Sets the proxy credentials
|
||||
# Accepted format:
|
||||
# username:password
|
||||
# { proxy_username: username, :proxy_password: password }
|
||||
#
|
||||
# @param [ String ] auth
|
||||
#
|
||||
# @return [ void ]
|
||||
def proxy_auth=(auth)
|
||||
unless auth.nil?
|
||||
if auth.is_a?(Hash) && auth.include?(:proxy_username) && auth.include?(:proxy_password)
|
||||
@proxy_auth = auth[:proxy_username] + ':' + auth[:proxy_password]
|
||||
elsif auth.is_a?(String) && auth.index(':') != nil
|
||||
@proxy_auth = auth
|
||||
else
|
||||
raise invalid_proxy_auth_format
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def invalid_proxy_auth_format
|
||||
'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}'
|
||||
end
|
||||
|
||||
# Override with the options if they are set
|
||||
# @param [ Hash ] options
|
||||
#
|
||||
# @return [ void ]
|
||||
def override_config(options = {})
|
||||
options.each do |option, value|
|
||||
if value != nil and OPTIONS.include?(option)
|
||||
self.send(:"#{option}=", value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
@@ -67,7 +67,7 @@ class WpItems < Array
|
||||
results = new
|
||||
item_class = self.item_class
|
||||
type = self.to_s.gsub(/Wp/, '').downcase
|
||||
response = Browser.instance.get(wp_target.url)
|
||||
response = Browser.get(wp_target.url)
|
||||
item_options = {
|
||||
wp_content_dir: wp_target.wp_content_dir,
|
||||
wp_plugins_dir: wp_target.wp_plugins_dir,
|
||||
|
||||
@@ -10,7 +10,7 @@ class WpUsers < WpItems
|
||||
max_display_name_length = self.sort { |a, b| a.display_name.length <=> b.display_name.length }.last.display_name.length
|
||||
|
||||
inner_space = 2
|
||||
id_length = (max_id_length + inner_space * 2) /2 *2
|
||||
id_length = (max_id_length + inner_space * 2) /2 * 2
|
||||
login_length = max_login_length + inner_space * 2
|
||||
display_name_length = max_display_name_length + inner_space * 2
|
||||
|
||||
|
||||
@@ -107,3 +107,8 @@ def xml(file)
|
||||
config.noblanks
|
||||
end
|
||||
end
|
||||
|
||||
def redefine_constant(constant, value)
|
||||
Object.send(:remove_const, constant)
|
||||
Object.const_set(constant, value)
|
||||
end
|
||||
|
||||
@@ -13,7 +13,7 @@ class WpItem
|
||||
# @return [ Boolean ]
|
||||
def exists?(options = {}, response = nil)
|
||||
unless response
|
||||
response = Browser.instance.get(url)
|
||||
response = Browser.get(url)
|
||||
end
|
||||
exists_from_response?(response, options)
|
||||
end
|
||||
|
||||
@@ -7,7 +7,7 @@ class WpItem
|
||||
|
||||
# @return [ Boolean ]
|
||||
def has_readme?
|
||||
Browser.instance.get(readme_url).code == 200 ? true : false
|
||||
Browser.get(readme_url).code == 200 ? true : false
|
||||
end
|
||||
|
||||
# @return [ String ] The url to the readme file
|
||||
@@ -17,7 +17,7 @@ class WpItem
|
||||
|
||||
# @return [ Boolean ]
|
||||
def has_changelog?
|
||||
Browser.instance.get(changelog_url).code == 200 ? true : false
|
||||
Browser.get(changelog_url).code == 200 ? true : false
|
||||
end
|
||||
|
||||
# @return [ String ] The url to the changelog file
|
||||
@@ -27,7 +27,7 @@ class WpItem
|
||||
|
||||
# @return [ Boolean ]
|
||||
def has_directory_listing?
|
||||
Browser.instance.get(@uri.to_s).body[%r{<title>Index of}] ? true : false
|
||||
Browser.get(@uri.to_s).body[%r{<title>Index of}] ? true : false
|
||||
end
|
||||
|
||||
# Discover any error_log files created by WordPress
|
||||
@@ -41,7 +41,7 @@ class WpItem
|
||||
#
|
||||
# @return [ Boolean ]
|
||||
def has_error_log?
|
||||
response_body = Browser.instance.get(error_log_url, headers: {'range' => 'bytes=0-700'}).body
|
||||
response_body = Browser.get(error_log_url, headers: {'range' => 'bytes=0-700'}).body
|
||||
response_body[%r{PHP Fatal error}i] ? true : false
|
||||
end
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ class WpItem
|
||||
# @return [ String ] The version number
|
||||
def version
|
||||
unless @version
|
||||
response = Browser.instance.get(readme_url)
|
||||
response = Browser.get(readme_url)
|
||||
@version = response.body[%r{stable tag: #{WpVersion.version_pattern}}i, 1]
|
||||
end
|
||||
@version
|
||||
|
||||
@@ -27,7 +27,7 @@ class WpTheme < WpItem
|
||||
#
|
||||
# @return [ WpTheme ]
|
||||
def find_from_css_link(target_uri)
|
||||
response = Browser.instance.get_and_follow_location(target_uri.to_s)
|
||||
response = Browser.get_and_follow_location(target_uri.to_s)
|
||||
|
||||
# https + domain is optional because of relative links
|
||||
matches = %r{(?:https?://[^"']+)?/([^/]+)/themes/([^"']+)/style.css}i.match(response.body)
|
||||
@@ -49,7 +49,7 @@ class WpTheme < WpItem
|
||||
#
|
||||
# @return [ WpTheme ]
|
||||
def find_from_wooframework(target_uri)
|
||||
body = Browser.instance.get(target_uri.to_s).body
|
||||
body = Browser.get(target_uri.to_s).body
|
||||
regexp = %r{<meta name="generator" content="([^\s"]+)\s?([^"]+)?" />\s+<meta name="generator" content="WooFramework\s?([^"]+)?" />}
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ class WpTheme < WpItem
|
||||
|
||||
def version
|
||||
unless @version
|
||||
@version = Browser.instance.get(style_url).body[%r{Version:\s([^\s]+)}i, 1]
|
||||
@version = Browser.get(style_url).body[%r{Version:\s([^\s]+)}i, 1]
|
||||
|
||||
# Get Version from readme.txt
|
||||
@version ||= super
|
||||
|
||||
@@ -9,7 +9,7 @@ class WpTimthumb < WpItem
|
||||
# @return [ String ] The version
|
||||
def version
|
||||
unless @version
|
||||
response = Browser.instance.get(url)
|
||||
response = Browser.get(url)
|
||||
@version = response.body[%r{TimThumb version\s*: ([^<]+)} , 1]
|
||||
end
|
||||
@version
|
||||
|
||||
@@ -24,7 +24,7 @@ class WpUser < WpItem
|
||||
|
||||
@login = Existable.login_from_author_pattern(location)
|
||||
@display_name = Existable.display_name_from_body(
|
||||
Browser.instance.get(location).body
|
||||
Browser.get(location).body
|
||||
)
|
||||
elsif response.code == 200 # login in body?
|
||||
@login = Existable.login_from_body(response.body)
|
||||
|
||||
@@ -45,7 +45,7 @@ class WpVersion < WpItem
|
||||
# @return [ String ]
|
||||
def scan_url(target_uri, pattern, path = nil)
|
||||
url = path ? target_uri.merge(path).to_s : target_uri.to_s
|
||||
response = Browser.instance.get_and_follow_location(url)
|
||||
response = Browser.get_and_follow_location(url)
|
||||
|
||||
response.body[pattern, 1]
|
||||
end
|
||||
@@ -163,7 +163,7 @@ class WpVersion < WpItem
|
||||
xml.xpath('//file').each do |node|
|
||||
wp_item.path = node.attribute('src').text
|
||||
|
||||
response = Browser.instance.get(wp_item.url)
|
||||
response = Browser.get(wp_item.url)
|
||||
md5sum = Digest::MD5.hexdigest(response.body)
|
||||
|
||||
node.search('hash').each do |hash|
|
||||
|
||||
@@ -33,7 +33,7 @@ begin
|
||||
rescue LoadError => e
|
||||
puts "[ERROR] #{e}"
|
||||
|
||||
missing_gem = e.to_s[%r{ -- ([^\z/]+)/?}, 1]
|
||||
missing_gem = e.to_s[%r{ -- ([^/]+)/?\z}, 1]
|
||||
if missing_gem
|
||||
if missing_gem =~ /nokogiri/i
|
||||
puts
|
||||
|
||||
@@ -18,11 +18,11 @@ class WebSite
|
||||
|
||||
# Checks if the remote website is up.
|
||||
def online?
|
||||
Browser.instance.get(@uri.to_s).code != 0
|
||||
Browser.get(@uri.to_s).code != 0
|
||||
end
|
||||
|
||||
def has_basic_auth?
|
||||
Browser.instance.get(@uri.to_s).code == 401
|
||||
Browser.get(@uri.to_s).code == 401
|
||||
end
|
||||
|
||||
def has_xml_rpc?
|
||||
@@ -38,7 +38,7 @@ class WebSite
|
||||
end
|
||||
|
||||
def xml_rpc_url_from_headers
|
||||
headers = Browser.instance.get(@uri.to_s).headers_hash
|
||||
headers = Browser.get(@uri.to_s).headers_hash
|
||||
xmlrpc_url = nil
|
||||
|
||||
unless headers.nil?
|
||||
@@ -51,7 +51,7 @@ class WebSite
|
||||
end
|
||||
|
||||
def xml_rpc_url_from_body
|
||||
body = Browser.instance.get(@uri.to_s).body
|
||||
body = Browser.get(@uri.to_s).body
|
||||
|
||||
body[%r{<link rel="pingback" href="([^"]+)" ?\/?>}, 1]
|
||||
end
|
||||
@@ -62,7 +62,7 @@ class WebSite
|
||||
def redirection(url = nil)
|
||||
redirection = nil
|
||||
url ||= @uri.to_s
|
||||
response = Browser.instance.get(url)
|
||||
response = Browser.get(url)
|
||||
|
||||
if response.code == 301 || response.code == 302
|
||||
redirection = response.headers_hash['location']
|
||||
@@ -78,7 +78,7 @@ class WebSite
|
||||
|
||||
# Return the MD5 hash of the page given by url
|
||||
def self.page_hash(url)
|
||||
Digest::MD5.hexdigest(Browser.instance.get(url).body)
|
||||
Digest::MD5.hexdigest(Browser.get(url).body)
|
||||
end
|
||||
|
||||
def homepage_hash
|
||||
@@ -100,13 +100,13 @@ class WebSite
|
||||
# Will try to find the rss url in the homepage
|
||||
# Only the first one found iw returned
|
||||
def rss_url
|
||||
homepage_body = Browser.instance.get(@uri.to_s).body
|
||||
homepage_body = Browser.get(@uri.to_s).body
|
||||
homepage_body[%r{<link .* type="application/rss\+xml" .* href="([^"]+)" />}, 1]
|
||||
end
|
||||
|
||||
# Checks if a robots.txt file exists
|
||||
def has_robots?
|
||||
Browser.instance.get(robots_url).code == 200
|
||||
Browser.get(robots_url).code == 200
|
||||
end
|
||||
|
||||
# Gets a robots.txt URL
|
||||
|
||||
@@ -11,14 +11,14 @@ require 'wp_target/wp_custom_directories'
|
||||
require 'wp_target/wp_full_path_disclosure'
|
||||
|
||||
class WpTarget < WebSite
|
||||
include Malwares
|
||||
include WpReadme
|
||||
include BruteForce
|
||||
include WpRegistrable
|
||||
include WpConfigBackup
|
||||
include WpLoginProtection
|
||||
include WpCustomDirectories
|
||||
include WpFullPathDisclosure
|
||||
include WpTarget::Malwares
|
||||
include WpTarget::WpReadme
|
||||
include WpTarget::BruteForce
|
||||
include WpTarget::WpRegistrable
|
||||
include WpTarget::WpConfigBackup
|
||||
include WpTarget::WpLoginProtection
|
||||
include WpTarget::WpCustomDirectories
|
||||
include WpTarget::WpFullPathDisclosure
|
||||
|
||||
attr_reader :verbose
|
||||
|
||||
@@ -38,17 +38,17 @@ class WpTarget < WebSite
|
||||
def wordpress?
|
||||
wordpress = false
|
||||
|
||||
response = Browser.instance.get_and_follow_location(@uri.to_s)
|
||||
response = Browser.get_and_follow_location(@uri.to_s)
|
||||
|
||||
if response.body =~ /["'][^"']*\/wp-content\/[^"']*["']/i
|
||||
wordpress = true
|
||||
else
|
||||
response = Browser.instance.get_and_follow_location(xml_rpc_url)
|
||||
response = Browser.get_and_follow_location(xml_rpc_url)
|
||||
|
||||
if response.body =~ %r{XML-RPC server accepts POST requests only}i
|
||||
wordpress = true
|
||||
else
|
||||
response = Browser.instance.get_and_follow_location(login_url)
|
||||
response = Browser.get_and_follow_location(login_url)
|
||||
|
||||
if response.code == 200 && response.body =~ %r{WordPress}i
|
||||
wordpress = true
|
||||
@@ -104,7 +104,7 @@ class WpTarget < WebSite
|
||||
|
||||
def has_debug_log?
|
||||
# We only get the first 700 bytes of the file to avoid loading huge file (like 2Go)
|
||||
response_body = Browser.instance.get(debug_log_url(), headers: {'range' => 'bytes=0-700'}).body
|
||||
response_body = Browser.get(debug_log_url(), headers: {'range' => 'bytes=0-700'}).body
|
||||
response_body[%r{\[[^\]]+\] PHP (?:Warning|Error|Notice):}] ? true : false
|
||||
end
|
||||
|
||||
@@ -120,7 +120,7 @@ class WpTarget < WebSite
|
||||
end
|
||||
|
||||
def search_replace_db_2_exists?
|
||||
resp = Browser.instance.get(search_replace_db_2_url)
|
||||
resp = Browser.get(search_replace_db_2_url)
|
||||
resp.code == 200 && resp.body[%r{by interconnect}i]
|
||||
end
|
||||
end
|
||||
|
||||
@@ -17,7 +17,7 @@ class WpTarget < WebSite
|
||||
unless @malwares
|
||||
malwares_found = []
|
||||
malwares_file = Malwares.malwares_file(malwares_file_path)
|
||||
index_page_body = Browser.instance.get(@uri.to_s).body
|
||||
index_page_body = Browser.get(@uri.to_s).body
|
||||
|
||||
File.open(malwares_file, 'r') do |file|
|
||||
file.readlines.collect do |url|
|
||||
|
||||
@@ -6,7 +6,7 @@ class WpTarget < WebSite
|
||||
# @return [ String ] The wp-content directory
|
||||
def wp_content_dir
|
||||
unless @wp_content_dir
|
||||
index_body = Browser.instance.get(@uri.to_s).body
|
||||
index_body = Browser.get(@uri.to_s).body
|
||||
uri_path = @uri.path # Only use the path because domain can be text or an IP
|
||||
|
||||
if index_body[/\/wp-content\/(?:themes|plugins)\//i] || default_wp_content_dir_exists?
|
||||
@@ -22,7 +22,7 @@ class WpTarget < WebSite
|
||||
|
||||
# @return [ Boolean ]
|
||||
def default_wp_content_dir_exists?
|
||||
response = Browser.instance.get(@uri.merge('wp-content').to_s)
|
||||
response = Browser.get(@uri.merge('wp-content').to_s)
|
||||
hash = Digest::MD5.hexdigest(response.body)
|
||||
|
||||
if WpTarget.valid_response_codes.include?(response.code)
|
||||
@@ -42,7 +42,7 @@ class WpTarget < WebSite
|
||||
|
||||
# @return [ Boolean ]
|
||||
def wp_plugins_dir_exists?
|
||||
Browser.instance.get(@uri.merge(wp_plugins_dir)).code != 404
|
||||
Browser.get(@uri.merge(wp_plugins_dir).to_s).code != 404
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
@@ -7,7 +7,7 @@ class WpTarget < WebSite
|
||||
#
|
||||
# @return [ Boolean ]
|
||||
def has_full_path_disclosure?
|
||||
response = Browser.instance.get(full_path_disclosure_url())
|
||||
response = Browser.get(full_path_disclosure_url())
|
||||
response.body[%r{Fatal error}i] ? true : false
|
||||
end
|
||||
|
||||
|
||||
@@ -38,17 +38,17 @@ class WpTarget < WebSite
|
||||
# Thanks to Alip Aswalid for providing this method.
|
||||
# http://wordpress.org/extend/plugins/login-lockdown/
|
||||
def has_login_lockdown_protection?
|
||||
Browser.instance.get(login_url).body =~ %r{Login LockDown}i ? true : false
|
||||
Browser.get(login_url).body =~ %r{Login LockDown}i ? true : false
|
||||
end
|
||||
|
||||
# http://wordpress.org/extend/plugins/login-lock/
|
||||
def has_login_lock_protection?
|
||||
Browser.instance.get(login_url).body =~ %r{LOGIN LOCK} ? true : false
|
||||
Browser.get(login_url).body =~ %r{LOGIN LOCK} ? true : false
|
||||
end
|
||||
|
||||
# http://wordpress.org/extend/plugins/better-wp-security/
|
||||
def has_better_wp_security_protection?
|
||||
Browser.instance.get(better_wp_security_url).code != 404
|
||||
Browser.get(better_wp_security_url).code != 404
|
||||
end
|
||||
|
||||
def plugin_url(plugin_name)
|
||||
@@ -66,7 +66,7 @@ class WpTarget < WebSite
|
||||
|
||||
# http://wordpress.org/extend/plugins/simple-login-lockdown/
|
||||
def has_simple_login_lockdown_protection?
|
||||
Browser.instance.get(simple_login_lockdown_url).code != 404
|
||||
Browser.get(simple_login_lockdown_url).code != 404
|
||||
end
|
||||
|
||||
def simple_login_lockdown_url
|
||||
@@ -75,7 +75,7 @@ class WpTarget < WebSite
|
||||
|
||||
# http://wordpress.org/extend/plugins/login-security-solution/
|
||||
def has_login_security_solution_protection?
|
||||
Browser.instance.get(login_security_solution_url()).code != 404
|
||||
Browser.get(login_security_solution_url()).code != 404
|
||||
end
|
||||
|
||||
def login_security_solution_url
|
||||
@@ -84,7 +84,7 @@ class WpTarget < WebSite
|
||||
|
||||
# http://wordpress.org/extend/plugins/limit-login-attempts/
|
||||
def has_limit_login_attempts_protection?
|
||||
Browser.instance.get(limit_login_attempts_url).code != 404
|
||||
Browser.get(limit_login_attempts_url).code != 404
|
||||
end
|
||||
|
||||
def limit_login_attempts_url
|
||||
@@ -93,7 +93,7 @@ class WpTarget < WebSite
|
||||
|
||||
# http://wordpress.org/extend/plugins/bluetrait-event-viewer/
|
||||
def has_bluetrait_event_viewer_protection?
|
||||
Browser.instance.get(bluetrait_event_viewer_url).code != 404
|
||||
Browser.get(bluetrait_event_viewer_url).code != 404
|
||||
end
|
||||
|
||||
def bluetrait_event_viewer_url
|
||||
|
||||
@@ -10,7 +10,7 @@ class WpTarget < WebSite
|
||||
#
|
||||
# @return [ Boolean ]
|
||||
def has_readme?
|
||||
response = Browser.instance.get(readme_url())
|
||||
response = Browser.get(readme_url())
|
||||
|
||||
unless response.code == 404
|
||||
return response.body =~ %r{wordpress}i ? true : false
|
||||
|
||||
@@ -7,7 +7,7 @@ class WpTarget < WebSite
|
||||
#
|
||||
# @return [ Boolean ]
|
||||
def registration_enabled?
|
||||
resp = Browser.instance.get(registration_url)
|
||||
resp = Browser.get(registration_url)
|
||||
# redirect only on non multi sites
|
||||
if resp.code == 302 and resp.headers_hash['location'] =~ /wp-login\.php\?registration=disabled/i
|
||||
enabled = false
|
||||
@@ -34,8 +34,7 @@ class WpTarget < WebSite
|
||||
unless @multisite
|
||||
# when multi site, there is no redirection or a redirect to the site itself
|
||||
# otherwise redirect to wp-login.php
|
||||
url = @uri.merge('wp-signup.php')
|
||||
resp = Browser.instance.get(url)
|
||||
resp = Browser.get(@uri.merge('wp-signup.php').to_s)
|
||||
|
||||
if resp.code == 302 and resp.headers_hash['location'] =~ /wp-login\.php\?action=register/
|
||||
@multisite = false
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../common/common_helper')
|
||||
|
||||
require_files_from_directory(WPSCAN_LIB_DIR + '/modules')
|
||||
require_files_from_directory(WPSCAN_LIB_DIR, '**/*.rb')
|
||||
|
||||
# wpscan usage
|
||||
|
||||
@@ -3,156 +3,30 @@
|
||||
require 'spec_helper'
|
||||
|
||||
describe Browser do
|
||||
it_behaves_like 'Browser::Actions'
|
||||
it_behaves_like 'Browser::Options'
|
||||
|
||||
CONFIG_FILE_WITHOUT_PROXY = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf.json'
|
||||
CONFIG_FILE_WITH_PROXY = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf_proxy.json'
|
||||
CONFIG_FILE_WITH_PROXY_AND_AUTH = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf_proxy_auth.json'
|
||||
INSTANCE_VARS_TO_CHECK = ['user_agent', 'user_agent_mode', 'available_user_agents', 'proxy', 'max_threads', 'request_timeout', 'cache_ttl']
|
||||
#CONFIG_FILE_WITH_PROXY_AND_AUTH = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf_proxy_auth.json'
|
||||
|
||||
before :all do
|
||||
@json_config_without_proxy = JSON.parse(File.read(CONFIG_FILE_WITHOUT_PROXY))
|
||||
@json_config_with_proxy = JSON.parse(File.read(CONFIG_FILE_WITH_PROXY))
|
||||
end
|
||||
|
||||
before :each do
|
||||
Browser::reset
|
||||
@browser = Browser.instance(config_file: CONFIG_FILE_WITHOUT_PROXY)
|
||||
end
|
||||
subject(:browser) {
|
||||
Browser.reset
|
||||
Browser.instance(options)
|
||||
}
|
||||
let(:options) { {} }
|
||||
let(:instance_vars_to_check) {
|
||||
['user_agent', 'user_agent_mode', 'available_user_agents', 'proxy',
|
||||
'max_threads', 'cache_ttl']
|
||||
}
|
||||
let(:json_config_without_proxy) { JSON.parse(File.read(CONFIG_FILE_WITHOUT_PROXY)) }
|
||||
let(:json_config_with_proxy) { JSON.parse(File.read(CONFIG_FILE_WITH_PROXY)) }
|
||||
|
||||
def check_instance_variables(browser, json_expected_vars)
|
||||
json_expected_vars['max_threads'] ||= 1 # max_thread can not be nil
|
||||
|
||||
INSTANCE_VARS_TO_CHECK.each do |instance_variable_name|
|
||||
browser.send(:"#{instance_variable_name}").should === json_expected_vars[instance_variable_name]
|
||||
end
|
||||
end
|
||||
|
||||
describe '#user_agent_mode setter / getter' do
|
||||
# Testing all valid modes
|
||||
Browser::USER_AGENT_MODES.each do |user_agent_mode|
|
||||
it "should set / return #{user_agent_mode}" do
|
||||
@browser.user_agent_mode = user_agent_mode
|
||||
@browser.user_agent_mode.should === user_agent_mode
|
||||
end
|
||||
end
|
||||
|
||||
it "shoud set the mode to 'static' if nil is given" do
|
||||
@browser.user_agent_mode = nil
|
||||
@browser.user_agent_mode.should === 'static'
|
||||
end
|
||||
|
||||
it 'should raise an error if the mode in not valid' do
|
||||
expect { @browser.user_agent_mode = 'invalid-mode' }.to raise_error
|
||||
end
|
||||
end
|
||||
|
||||
describe '#max_threads=' do
|
||||
it 'should set max_threads to 1 if nil is given' do
|
||||
@browser.max_threads = nil
|
||||
@browser.max_threads.should === 1
|
||||
end
|
||||
|
||||
it 'should set max_threads to 1 if 0 is given' do
|
||||
@browser.max_threads = 0
|
||||
@browser.max_threads.should === 1
|
||||
end
|
||||
end
|
||||
|
||||
describe '#proxy_auth=' do
|
||||
after :each do
|
||||
if @raise_error
|
||||
expect { @browser.proxy_auth = @proxy_auth }.to raise_error
|
||||
else
|
||||
@browser.proxy_auth = @proxy_auth
|
||||
@browser.proxy_auth.should === @expected
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the auth supplied is' do
|
||||
|
||||
context 'not a String or a Hash' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = 10
|
||||
@raise_error = true
|
||||
end
|
||||
end
|
||||
|
||||
context 'a String with' do
|
||||
context 'invalid format' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = 'invaludauthformat'
|
||||
@raise_error = true
|
||||
end
|
||||
end
|
||||
|
||||
context 'valid format' do
|
||||
it 'sets the auth' do
|
||||
@proxy_auth = 'username:passwd'
|
||||
@expected = @proxy_auth
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'a Hash with' do
|
||||
context 'only :proxy_username' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = { proxy_username: 'username' }
|
||||
@raise_error = true
|
||||
end
|
||||
end
|
||||
|
||||
context 'only :proxy_password' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = { proxy_password: 'hello' }
|
||||
@raise_error = true
|
||||
end
|
||||
end
|
||||
|
||||
context ':proxy_username and :proxy_password' do
|
||||
it 'sets the auth' do
|
||||
@proxy_auth = { proxy_username: 'user', proxy_password: 'pass' }
|
||||
@expected = 'user:pass'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
describe '#user_agent' do
|
||||
available_user_agents = %w{ ua-1 ua-2 ua-3 ua-4 ua-6 ua-7 ua-8 ua-9 ua-10 ua-11 ua-12 ua-13 ua-14 ua-15 ua-16 ua-17 }
|
||||
|
||||
it 'should always return the same user agent in static mode' do
|
||||
@browser.user_agent = 'fake UA'
|
||||
@browser.user_agent_mode = 'static'
|
||||
|
||||
(1..3).each do
|
||||
@browser.user_agent.should === 'fake UA'
|
||||
end
|
||||
end
|
||||
|
||||
it 'should choose a random user_agent in the available_user_agents array an always return it' do
|
||||
@browser.available_user_agents = available_user_agents
|
||||
@browser.user_agent = 'Firefox 11.0'
|
||||
@browser.user_agent_mode = 'semi-static'
|
||||
|
||||
user_agent = @browser.user_agent
|
||||
user_agent.should_not === 'Firefox 11.0'
|
||||
available_user_agents.include?(user_agent).should be_true
|
||||
|
||||
(1..3).each do
|
||||
@browser.user_agent.should === user_agent
|
||||
end
|
||||
end
|
||||
|
||||
it 'should return a random user agent each time' do
|
||||
@browser.available_user_agents = available_user_agents
|
||||
@browser.user_agent_mode = 'random'
|
||||
|
||||
ua_1 = @browser.user_agent
|
||||
ua_2 = @browser.user_agent
|
||||
ua_3 = @browser.user_agent
|
||||
|
||||
fail if ua_1 === ua_2 and ua_2 === ua_3
|
||||
instance_vars_to_check.each do |variable_name|
|
||||
browser.send(:"#{variable_name}").should === json_expected_vars[variable_name]
|
||||
end
|
||||
end
|
||||
|
||||
@@ -162,64 +36,60 @@ describe Browser do
|
||||
end
|
||||
end
|
||||
|
||||
describe "#instance with :config_file = #{CONFIG_FILE_WITHOUT_PROXY}" do
|
||||
it 'will check the instance vars' do
|
||||
Browser.reset
|
||||
check_instance_variables(
|
||||
Browser.instance(config_file: CONFIG_FILE_WITHOUT_PROXY),
|
||||
@json_config_without_proxy
|
||||
)
|
||||
describe '::instance' do
|
||||
after { check_instance_variables(browser, @json_expected_vars) }
|
||||
|
||||
context "when default config_file = #{CONFIG_FILE_WITHOUT_PROXY}" do
|
||||
it 'will check the instance vars' do
|
||||
@json_expected_vars = json_config_without_proxy
|
||||
end
|
||||
end
|
||||
|
||||
context "when :config_file = #{CONFIG_FILE_WITH_PROXY}" do
|
||||
let(:options) { { config_file: CONFIG_FILE_WITH_PROXY } }
|
||||
|
||||
it 'will check the instance vars' do
|
||||
@json_expected_vars = json_config_with_proxy
|
||||
end
|
||||
end
|
||||
|
||||
context 'when options[:cache_dir]' do
|
||||
let(:cache_dir) { CACHE_DIR + '/somewhere' }
|
||||
let(:options) { { cache_dir: cache_dir } }
|
||||
|
||||
after { subject.cache_dir.should == cache_dir }
|
||||
|
||||
it 'sets @cache_dir' do
|
||||
@json_expected_vars = json_config_without_proxy
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "#instance with :config_file = #{CONFIG_FILE_WITH_PROXY}" do
|
||||
it 'will check the instance vars' do
|
||||
Browser.reset
|
||||
check_instance_variables(
|
||||
Browser.instance(config_file: CONFIG_FILE_WITH_PROXY),
|
||||
@json_config_with_proxy
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# TODO Write something to test all possible overriding
|
||||
describe 'override option : user_agent & threads' do
|
||||
it 'will check the instance vars, with an overriden one' do
|
||||
Browser.reset
|
||||
check_instance_variables(
|
||||
Browser.instance(
|
||||
config_file: CONFIG_FILE_WITHOUT_PROXY,
|
||||
user_agent: 'fake IE'
|
||||
),
|
||||
@json_config_without_proxy.merge('user_agent' => 'fake IE')
|
||||
)
|
||||
end
|
||||
|
||||
it 'should not override the max_threads if max_threads = nil' do
|
||||
Browser.reset
|
||||
check_instance_variables(
|
||||
Browser.instance(
|
||||
config_file: CONFIG_FILE_WITHOUT_PROXY,
|
||||
max_threads: nil
|
||||
),
|
||||
@json_config_without_proxy
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# TODO
|
||||
describe '#load_config' do
|
||||
it 'should raise an error if file is a symlink' do
|
||||
symlink = './rspec_symlink'
|
||||
browser = Browser.instance
|
||||
context 'when config_file is a symlink' do
|
||||
let(:config_file) { './rspec_symlink' }
|
||||
|
||||
File.symlink('./testfile', symlink)
|
||||
expect { browser.load_config(symlink) }.to raise_error("[ERROR] Config file is a symlink.")
|
||||
File.unlink(symlink)
|
||||
it 'raises an error' do
|
||||
File.symlink('./testfile', config_file)
|
||||
expect { browser.load_config(config_file) }.to raise_error("[ERROR] Config file is a symlink.")
|
||||
File.unlink(config_file)
|
||||
end
|
||||
end
|
||||
|
||||
context 'otherwise' do
|
||||
after do
|
||||
browser.load_config(@config_file)
|
||||
check_instance_variables(browser, @expected)
|
||||
end
|
||||
|
||||
it 'sets the correct variables' do
|
||||
@config_file = CONFIG_FILE_WITH_PROXY
|
||||
@expected = json_config_without_proxy.merge(json_config_with_proxy)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#append_params_header_field' do
|
||||
describe '::append_params_header_field' do
|
||||
after :each do
|
||||
Browser.append_params_header_field(
|
||||
@params,
|
||||
@@ -256,7 +126,6 @@ describe Browser do
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
describe '#merge_request_params' do
|
||||
@@ -272,10 +141,10 @@ describe Browser do
|
||||
}
|
||||
|
||||
after :each do
|
||||
@browser.stub(user_agent: 'SomeUA')
|
||||
@browser.cache_ttl = 250
|
||||
browser.stub(user_agent: 'SomeUA')
|
||||
browser.cache_ttl = 250
|
||||
|
||||
@browser.merge_request_params(params).should == @expected
|
||||
browser.merge_request_params(params).should == @expected
|
||||
end
|
||||
|
||||
it 'sets the User-Agent header field and cache_ttl' do
|
||||
@@ -288,27 +157,26 @@ describe Browser do
|
||||
let(:proxy_expectation) { default_expectation.merge(proxy: proxy) }
|
||||
|
||||
it 'merges the proxy' do
|
||||
@browser.proxy = proxy
|
||||
@expected = proxy_expectation
|
||||
browser.proxy = proxy
|
||||
@expected = proxy_expectation
|
||||
end
|
||||
|
||||
context 'when @proxy_auth' do
|
||||
it 'sets the proxy_auth' do
|
||||
@browser.proxy = proxy
|
||||
@browser.proxy_auth = 'user:pass'
|
||||
@expected = proxy_expectation.merge(proxyauth: 'user:pass')
|
||||
browser.proxy = proxy
|
||||
browser.proxy_auth = 'user:pass'
|
||||
@expected = proxy_expectation.merge(proxyauth: 'user:pass')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'when @basic_auth' do
|
||||
it 'appends the basic_auth' do
|
||||
@browser.basic_auth = 'basic-auth'
|
||||
browser.basic_auth = 'user:pass'
|
||||
@expected = default_expectation.merge(
|
||||
headers: default_expectation[:headers].merge('Authorization' => 'basic-auth')
|
||||
headers: default_expectation[:headers].merge('Authorization' => 'Basic '+Base64.encode64('user:pass').chomp)
|
||||
)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
context 'when the cache_ttl is alreday set' do
|
||||
@@ -318,67 +186,20 @@ describe Browser do
|
||||
@expected = default_expectation.merge(params)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
# TODO
|
||||
describe '#forge_request' do
|
||||
let(:url) { 'http://example.localhost' }
|
||||
|
||||
end
|
||||
it 'returns the correct Typhoeus::Request' do
|
||||
subject.stub(merge_request_params: { cache_ttl: 10 })
|
||||
|
||||
describe '#post' do
|
||||
it 'should return a Typhoeus::Response wth body = "Welcome Master" if login=master&password=itsme!' do
|
||||
url = 'http://example.com/'
|
||||
|
||||
stub_request(:post, url).with(body: { login: 'master', password: 'itsme!' }).
|
||||
to_return(status: 200, body: 'Welcome Master')
|
||||
|
||||
response = @browser.post(
|
||||
url,
|
||||
body: 'login=master&password=itsme!'
|
||||
#body: { login: 'master', password: 'hello' } # It's should be this line, but it fails
|
||||
)
|
||||
|
||||
response.should be_a Typhoeus::Response
|
||||
response.body.should == 'Welcome Master'
|
||||
request = subject.forge_request(url)
|
||||
request.should be_a Typhoeus::Request
|
||||
request.url.should == url
|
||||
request.cache_ttl.should == 10
|
||||
end
|
||||
end
|
||||
|
||||
describe '#get' do
|
||||
it "should return a Typhoeus::Response with body = 'Hello World !'" do
|
||||
url = 'http://example.com/'
|
||||
|
||||
stub_request(:get, url).
|
||||
to_return(status: 200, body: 'Hello World !')
|
||||
|
||||
response = @browser.get(url)
|
||||
|
||||
response.should be_a Typhoeus::Response
|
||||
response.body.should == 'Hello World !'
|
||||
end
|
||||
end
|
||||
|
||||
describe '#get_and_follow_location' do
|
||||
# Typhoeus does not follow the location (maybe it's fixed in > 0.4.2)
|
||||
# Or, something else is wrong
|
||||
|
||||
#context 'whitout max_redirects params' do
|
||||
# context 'when multiples redirection' do
|
||||
# it 'returns the last redirection response' do
|
||||
# url = 'http://target.com'
|
||||
# first_redirection = 'www.first-redirection.com'
|
||||
# last_redirection = 'last-redirection.com'
|
||||
|
||||
# stub_request(:get, url).to_return(status: 301, headers: { location: first_redirection })
|
||||
# stub_request(:get, first_redirection).to_return(status: 301, headers: { location: last_redirection })
|
||||
# stub_request(:get, last_redirection).to_return(status: 200, body: 'Hello World!')
|
||||
|
||||
# response = @browser.get_and_follow_location(url)
|
||||
|
||||
# response.body.should === 'Hellow World!'
|
||||
# end
|
||||
# end
|
||||
#end
|
||||
end
|
||||
|
||||
describe 'testing caching' do
|
||||
@@ -386,11 +207,10 @@ describe Browser do
|
||||
|
||||
url = 'http://example.localhost'
|
||||
|
||||
stub_request(:get, url).
|
||||
to_return(status: 200, body: 'Hello World !')
|
||||
stub_request(:get, url).to_return(status: 200, body: 'Hello World !')
|
||||
|
||||
response1 = @browser.get(url)
|
||||
response2 = @browser.get(url)
|
||||
response1 = Browser.get(url)
|
||||
response2 = Browser.get(url)
|
||||
|
||||
response1.body.should == response2.body
|
||||
#WebMock.should have_requested(:get, url).times(1) # This one fail, dunno why :s (but it works without mock)
|
||||
@@ -401,9 +221,9 @@ describe Browser do
|
||||
it 'should not throw an encoding exception' do
|
||||
url = SPEC_FIXTURES_DIR + '/utf8.html'
|
||||
stub_request(:get, url).to_return(status: 200, body: File.read(url))
|
||||
response1 = @browser.get(url)
|
||||
expect { response1.body }.to_not raise_error
|
||||
|
||||
response = Browser.get(url)
|
||||
expect { response.body }.to_not raise_error
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
60
spec/shared_examples/browser/actions.rb
Normal file
60
spec/shared_examples/browser/actions.rb
Normal file
@@ -0,0 +1,60 @@
|
||||
# encoding: UTF-8
|
||||
|
||||
shared_examples 'Browser::Actions' do
|
||||
|
||||
describe '#post' do
|
||||
it 'returns a Typhoeus::Response wth body = "Welcome Master" if login=master&password=itsme!' do
|
||||
url = 'http://example.com/'
|
||||
|
||||
stub_request(:post, url).with(body: { login: 'master', password: 'itsme!' }).
|
||||
to_return(status: 200, body: 'Welcome Master')
|
||||
|
||||
response = Browser.post(
|
||||
url,
|
||||
body: 'login=master&password=itsme!'
|
||||
#body: { login: 'master', password: 'hello' } # It's should be this line, but it fails
|
||||
)
|
||||
|
||||
response.should be_a Typhoeus::Response
|
||||
response.body.should == 'Welcome Master'
|
||||
end
|
||||
end
|
||||
|
||||
describe '#get' do
|
||||
it "returns a Typhoeus::Response with body = 'Hello World !'" do
|
||||
url = 'http://example.com/'
|
||||
|
||||
stub_request(:get, url).
|
||||
to_return(status: 200, body: 'Hello World !')
|
||||
|
||||
response = Browser.get(url)
|
||||
|
||||
response.should be_a Typhoeus::Response
|
||||
response.body.should == 'Hello World !'
|
||||
end
|
||||
end
|
||||
|
||||
describe '#get_and_follow_location' do
|
||||
# Typhoeus does not follow the location with rspec
|
||||
# See https://github.com/typhoeus/typhoeus/issues/279
|
||||
|
||||
#context 'whitout max_redirects params' do
|
||||
# context 'when multiples redirection' do
|
||||
# it 'returns the last redirection response' do
|
||||
# url = 'http://target.com'
|
||||
# first_redirection = 'www.first-redirection.com'
|
||||
# last_redirection = 'last-redirection.com'
|
||||
|
||||
# stub_request(:get, url).to_return(status: 301, headers: { location: first_redirection })
|
||||
# stub_request(:get, first_redirection).to_return(status: 301, headers: { location: last_redirection })
|
||||
# stub_request(:get, last_redirection).to_return(status: 200, body: 'Hello World!')
|
||||
|
||||
# response = Browser.get_and_follow_location(url)
|
||||
|
||||
# response.body.should === 'Hellow World!'
|
||||
# end
|
||||
# end
|
||||
#end
|
||||
end
|
||||
|
||||
end
|
||||
262
spec/shared_examples/browser/options.rb
Normal file
262
spec/shared_examples/browser/options.rb
Normal file
@@ -0,0 +1,262 @@
|
||||
# encoding: UTF-8
|
||||
|
||||
shared_examples 'Browser::Options' do
|
||||
|
||||
describe '#basic_auth=' do
|
||||
let(:exception) { 'Invalid basic authentication format, "login:password" or "Basic base_64_encoded" expected' }
|
||||
|
||||
after do
|
||||
if @expected
|
||||
browser.basic_auth = @auth
|
||||
browser.basic_auth.should == @expected
|
||||
else
|
||||
expect { browser.basic_auth = @auth }.to raise_error(exception)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when invalid format' do
|
||||
it 'raises an error' do
|
||||
@auth = 'invalid'
|
||||
end
|
||||
end
|
||||
|
||||
context 'when login:password' do
|
||||
it 'sets the basic auth' do
|
||||
@auth = 'admin:weakpass'
|
||||
@expected = 'Basic YWRtaW46d2Vha3Bhc3M='
|
||||
end
|
||||
end
|
||||
|
||||
context 'when Basic base_64_encoded' do
|
||||
context 'when invalid base_64_encoded' do
|
||||
it 'raises an error' do
|
||||
@auth = 'Basic <script>alert(1)</script>'
|
||||
end
|
||||
end
|
||||
|
||||
it 'sets the basic auth' do
|
||||
@auth = 'Basic dXNlcm5hbWU6dGhlYmlncGFzc3dvcmRzb3dlYWs='
|
||||
@expected = @auth
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#max_threads= & #max_threads' do
|
||||
let(:exception) { 'max_threads must be an Integer > 0' }
|
||||
|
||||
after do
|
||||
if @expected
|
||||
browser.max_threads = @max_threads
|
||||
browser.max_threads.should == @expected
|
||||
else
|
||||
expect { browser.max_threads = @max_threads }.to raise_error(exception)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the argument is not an Integer > 0' do
|
||||
it 'raises an error' do
|
||||
@max_thrads = nil
|
||||
end
|
||||
|
||||
it 'raises an error' do
|
||||
@max_threads = -3
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the argument is an Integer' do
|
||||
it 'returns the @max_threads' do
|
||||
@max_threads = 10
|
||||
@expected = 10
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#user_agent_mode= & #user_agent_mode' do
|
||||
# Testing all valid modes
|
||||
Browser::USER_AGENT_MODES.each do |user_agent_mode|
|
||||
it "sets & returns #{user_agent_mode}" do
|
||||
browser.user_agent_mode = user_agent_mode
|
||||
browser.user_agent_mode.should === user_agent_mode
|
||||
end
|
||||
end
|
||||
|
||||
it 'sets the mode to "static" if nil is given' do
|
||||
browser.user_agent_mode = nil
|
||||
browser.user_agent_mode.should === 'static'
|
||||
end
|
||||
|
||||
it 'raises an error if the mode is not valid' do
|
||||
expect { browser.user_agent_mode = 'invalid-mode' }.to raise_error
|
||||
end
|
||||
end
|
||||
|
||||
describe '#user_agent= & #user_agent' do
|
||||
let(:available_user_agents) { %w{ ua-1 ua-2 ua-3 ua-4 ua-6 ua-7 ua-8 ua-9 ua-10 ua-11 ua-12 ua-13 ua-14 ua-15 ua-16 ua-17 } }
|
||||
|
||||
context 'when static mode' do
|
||||
it 'returns the same user agent' do
|
||||
browser.user_agent = 'fake UA'
|
||||
browser.user_agent_mode = 'static'
|
||||
|
||||
(1..3).each do
|
||||
browser.user_agent.should === 'fake UA'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'when semi-static mode' do
|
||||
it 'chooses a random user_agent in the available_user_agents array and always return it' do
|
||||
browser.available_user_agents = available_user_agents
|
||||
browser.user_agent = 'Firefox 11.0'
|
||||
browser.user_agent_mode = 'semi-static'
|
||||
|
||||
user_agent = browser.user_agent
|
||||
user_agent.should_not === 'Firefox 11.0'
|
||||
available_user_agents.include?(user_agent).should be_true
|
||||
|
||||
(1..3).each do
|
||||
browser.user_agent.should === user_agent
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'when random' do
|
||||
it 'returns a random user agent each time' do
|
||||
browser.available_user_agents = available_user_agents
|
||||
browser.user_agent_mode = 'random'
|
||||
|
||||
ua_1 = browser.user_agent
|
||||
ua_2 = browser.user_agent
|
||||
ua_3 = browser.user_agent
|
||||
|
||||
fail if ua_1 === ua_2 and ua_2 === ua_3
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'proxy=' do
|
||||
let(:exception) { 'Invalid proxy format. Should be [protocol://]host:port.' }
|
||||
|
||||
after do
|
||||
if @expected
|
||||
browser.proxy = @proxy
|
||||
browser.proxy.should == @expected
|
||||
else
|
||||
expect { browser.proxy = @proxy }.to raise_error(exception)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when invalid format' do
|
||||
it 'raises an error' do
|
||||
@proxy = 'yolo'
|
||||
end
|
||||
end
|
||||
|
||||
context 'when valid format' do
|
||||
@proxy = '127.0.0.1:9050'
|
||||
@expected = @proxy
|
||||
end
|
||||
end
|
||||
|
||||
describe 'proxy_auth=' do
|
||||
let(:exception) { 'Invalid proxy auth format, expected username:password or {proxy_username: username, proxy_password: password}' }
|
||||
|
||||
after :each do
|
||||
if @expected
|
||||
browser.proxy_auth = @proxy_auth
|
||||
browser.proxy_auth.should === @expected
|
||||
else
|
||||
expect { browser.proxy_auth = @proxy_auth }.to raise_error
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the auth supplied is' do
|
||||
context 'not a String or a Hash' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = 10
|
||||
end
|
||||
end
|
||||
|
||||
context 'a String with' do
|
||||
context 'invalid format' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = 'invaludauthformat'
|
||||
end
|
||||
end
|
||||
|
||||
context 'valid format' do
|
||||
it 'sets the auth' do
|
||||
@proxy_auth = 'username:passwd'
|
||||
@expected = @proxy_auth
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'a Hash with' do
|
||||
context 'only :proxy_username' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = { proxy_username: 'username' }
|
||||
end
|
||||
end
|
||||
|
||||
context 'only :proxy_password' do
|
||||
it 'raises an error' do
|
||||
@proxy_auth = { proxy_password: 'hello' }
|
||||
end
|
||||
end
|
||||
|
||||
context ':proxy_username and :proxy_password' do
|
||||
it 'sets the auth' do
|
||||
@proxy_auth = { proxy_username: 'user', proxy_password: 'pass' }
|
||||
@expected = 'user:pass'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#override_config' do
|
||||
after do
|
||||
browser.send(:override_config, override_options)
|
||||
end
|
||||
|
||||
let(:config) { JSON.parse(File.read(browser.config_file)) }
|
||||
|
||||
context 'when an option value is nil' do
|
||||
let(:override_options) { { max_threads: nil } }
|
||||
|
||||
it 'does not set it' do
|
||||
browser.should_not_receive(:max_threads=)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when an option is no allowed' do
|
||||
let(:override_options) { { not_allowed: 'owned' } }
|
||||
|
||||
it 'does not set it' do
|
||||
browser.should_not_receive(:not_allowed=)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when valid option' do
|
||||
let(:override_options) { { max_threads: 30 } }
|
||||
|
||||
it 'sets it' do
|
||||
browser.should_receive(:max_threads=).with(30)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when multiple options' do
|
||||
let(:override_options) {
|
||||
{ max_threads: 10, not_allowed: 'owned', proxy: 'host:port' }
|
||||
}
|
||||
|
||||
it 'sets @max_threads, @proxy' do
|
||||
browser.should_not_receive(:not_allowed=)
|
||||
browser.should_receive(:max_threads=).with(10)
|
||||
browser.should_receive(:proxy=).with('host:port')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
@@ -7,7 +7,7 @@ shared_examples 'WpItem::Existable' do
|
||||
let(:response) { Typhoeus::Response.new }
|
||||
|
||||
it 'does not create a request' do
|
||||
Browser.instance.should_not_receive(:get)
|
||||
Browser.should_not_receive(:get)
|
||||
subject.stub(:exists_from_response?).and_return(true)
|
||||
|
||||
subject.exists?({}, response).should be_true
|
||||
@@ -16,7 +16,7 @@ shared_examples 'WpItem::Existable' do
|
||||
|
||||
context 'when the response is not supplied' do
|
||||
it 'creates a request' do
|
||||
Browser.instance.should_receive(:get)
|
||||
Browser.should_receive(:get)
|
||||
subject.stub(:exists_from_response?).and_return(false)
|
||||
|
||||
subject.exists?.should be_false
|
||||
|
||||
@@ -27,19 +27,19 @@ shared_examples 'WpTarget::WpRegistrable' do
|
||||
describe '#registration_enabled?' do
|
||||
after do
|
||||
wp_target.stub(:multisite?).and_return(multisite)
|
||||
stub_request(:get, wp_target.registration_url.to_s).to_return(@stub)
|
||||
stub_request(:get, wp_target.registration_url).to_return(@stub)
|
||||
|
||||
wp_target.registration_enabled?.should === @expected
|
||||
end
|
||||
|
||||
context 'when multisite' do
|
||||
let(:multisite) { true }
|
||||
it 'returns false (multisite)' do
|
||||
it 'returns false' do
|
||||
@stub = { status: 302, headers: { 'Location' => 'wp-login.php?registration=disabled' } }
|
||||
@expected = false
|
||||
end
|
||||
|
||||
it 'returns true (multisite)' do
|
||||
it 'returns true' do
|
||||
@stub = { status: 200, body: %{<form id="setupform" method="post" action="wp-signup.php">} }
|
||||
@expected = true
|
||||
end
|
||||
@@ -48,12 +48,12 @@ shared_examples 'WpTarget::WpRegistrable' do
|
||||
context 'when not multisite' do
|
||||
let(:multisite) { false }
|
||||
|
||||
it 'returns false (not multisite)' do
|
||||
it 'returns false' do
|
||||
@stub = { status: 302, headers: { 'Location' => 'wp-login.php?registration=disabled' } }
|
||||
@expected = false
|
||||
end
|
||||
|
||||
it 'returns true (not multisite)' do
|
||||
it 'returns true' do
|
||||
@stub = { status: 200, body: %{<form name="registerform" id="registerform" action="wp-login.php"} }
|
||||
@expected = true
|
||||
end
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# encoding: UTF-8
|
||||
|
||||
# https://github.com/bblimke/webmock
|
||||
# https://github.com/colszowka/simplecov
|
||||
|
||||
require 'webmock/rspec'
|
||||
# Code Coverage (only works with ruby >= 1.9)
|
||||
require 'simplecov' if RUBY_VERSION >= '1.9'
|
||||
@@ -11,12 +8,15 @@ require File.expand_path(File.dirname(__FILE__) + '/../lib/common/common_helper'
|
||||
|
||||
SPEC_DIR = ROOT_DIR + '/spec'
|
||||
SPEC_LIB_DIR = SPEC_DIR + '/lib'
|
||||
SPEC_CACHE_DIR = SPEC_DIR + '/cache'
|
||||
SPEC_CACHE_DIR = SPEC_DIR + '/cache' # FIXME remove it
|
||||
SPEC_FIXTURES_DIR = SPEC_DIR + '/samples'
|
||||
SHARED_EXAMPLES_DIR = SPEC_DIR + '/shared_examples'
|
||||
SPEC_FIXTURES_CONF_DIR = SPEC_FIXTURES_DIR + '/conf'
|
||||
SPEC_FIXTURES_CONF_DIR = SPEC_FIXTURES_DIR + '/conf' # FIXME Remove it
|
||||
SPEC_FIXTURES_WP_VERSIONS_DIR = SPEC_FIXTURES_DIR + '/wp_versions'
|
||||
|
||||
redefine_constant(:CACHE_DIR, SPEC_DIR + '/cache')
|
||||
redefine_constant(:CONF_DIR, SPEC_FIXTURES_DIR + '/conf/browser') # FIXME Remove the /browser
|
||||
|
||||
MODELS_FIXTURES = SPEC_FIXTURES_DIR + '/common/models'
|
||||
COLLECTIONS_FIXTURES = SPEC_FIXTURES_DIR + '/common/collections'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user