From 8b9aec468a025caaf845604606dba6e1a635a8ad Mon Sep 17 00:00:00 2001 From: erwanlr Date: Wed, 20 Feb 2013 17:34:17 +0100 Subject: [PATCH] Initial work --- Gemfile | 4 +- lib/common/browser.rb | 55 ++++--------------- lib/common/cache_file_store.rb | 4 +- lib/environment.rb | 1 - lib/wpscan/modules/wp_usernames.rb | 2 +- lib/wpscan/wp_enumerator.rb | 2 +- lib/wpscan/wp_theme.rb | 2 +- .../plugins/checker/checker_plugin.rb | 2 +- spec/lib/common/browser_spec.rb | 20 ------- spec/lib/common/cache_file_store_spec.rb | 10 ++-- spec/spec_helper.rb | 2 +- 11 files changed, 26 insertions(+), 78 deletions(-) diff --git a/Gemfile b/Gemfile index 9092ebd9..5316b28b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,11 +1,11 @@ source "https://rubygems.org" -gem "typhoeus", "0.4.2" +gem "typhoeus", "~>0.6.1" gem "nokogiri" gem "json" group :development, :test do - gem "webmock", "1.8.11" + gem "webmock", "~>1.9.3" gem "simplecov" gem "rspec", :require => "spec" end diff --git a/lib/common/browser.rb b/lib/common/browser.rb index d4d9d17d..2de66412 100644 --- a/lib/common/browser.rb +++ b/lib/common/browser.rb @@ -48,7 +48,7 @@ class Browser @hydra = Typhoeus::Hydra.new( max_concurrency: @max_threads, - timeout: @request_timeout + #connecttimeout: @request_timeout ) # TODO : add an option for the cache dir instead of using a constant @@ -56,8 +56,7 @@ class Browser @cache.clean - # might be in CacheFileStore - setup_cache_handlers + #Typhoeus::Config.cache = @cache end private_class_method :new @@ -146,24 +145,6 @@ class Browser end end - def setup_cache_handlers - @hydra.cache_setter do |request| - @cache.write_entry( - Browser.generate_cache_key_from_request(request), - request.response, - request.cache_timeout - ) - end - - @hydra.cache_getter do |request| - @cache.read_entry( - Browser.generate_cache_key_from_request(request) - ) rescue nil - end - end - - private :setup_cache_handlers - def get(url, params = {}) run_request( forge_request(url, params.merge(method: :get)) @@ -177,10 +158,10 @@ class Browser end def get_and_follow_location(url, params = {}) - params[:max_redirects] ||= 2 + params[:maxredirs] ||= 2 run_request( - forge_request(url, params.merge(method: :get, follow_location: true)) + forge_request(url, params.merge(method: :get, followlocation: true)) ) end @@ -208,12 +189,13 @@ class Browser end end - unless params.has_key?(:disable_ssl_host_verification) - params = params.merge(:disable_ssl_host_verification => true) + # TODO : check if it's the default value into ethon. If so, removed the lines from here + unless params.has_key?(:ssl_verifyhost) + params = params.merge(ssl_verifyhost: 0) end - unless params.has_key?(:disable_ssl_peer_verification) - params = params.merge(:disable_ssl_peer_verification => true) + unless params.has_key?(:ssl_verifypeer) + params = params.merge(ssl_verifypeer: false) end if !params.has_key?(:headers) @@ -223,9 +205,9 @@ class Browser end # Used to enable the cache system if :cache_timeout > 0 - unless params.has_key?(:cache_timeout) - params = params.merge(:cache_timeout => @cache_timeout) - end + #unless params.has_key?(:cache_ttl) + # params = params.merge(cache_ttl: @cache_timeout) + #end params end @@ -247,17 +229,4 @@ class Browser end end end - - # The Typhoeus::Request.cache_key only hash the url :/ - # this one will include the params - # TODO : include also the method (:get, :post, :any) - def self.generate_cache_key_from_request(request) - cache_key = request.cache_key - - if request.params - cache_key = Digest::SHA1.hexdigest("#{cache_key}-#{request.params.hash}") - end - - cache_key - end end diff --git a/lib/common/cache_file_store.rb b/lib/common/cache_file_store.rb index 71bf33c5..e8f0a3cb 100644 --- a/lib/common/cache_file_store.rb +++ b/lib/common/cache_file_store.rb @@ -51,7 +51,7 @@ class CacheFileStore end end - def read_entry(key) + def get(key) entry_file_path = get_entry_file_path(key) if File.exists?(entry_file_path) @@ -59,7 +59,7 @@ class CacheFileStore end end - def write_entry(key, data_to_store, cache_timeout) + def set(key, data_to_store, cache_timeout) if cache_timeout > 0 File.open(get_entry_file_path(key), 'w') do |f| f.write(@serializer.dump(data_to_store)) diff --git a/lib/environment.rb b/lib/environment.rb index 242cc96c..dd9c209f 100644 --- a/lib/environment.rb +++ b/lib/environment.rb @@ -33,7 +33,6 @@ begin require 'rbconfig' require 'pp' # Third party libs - gem 'typhoeus', '=0.4.2' require 'typhoeus' require 'json' require 'nokogiri' diff --git a/lib/wpscan/modules/wp_usernames.rb b/lib/wpscan/modules/wp_usernames.rb index 94f421d7..8ba9b6e8 100644 --- a/lib/wpscan/modules/wp_usernames.rb +++ b/lib/wpscan/modules/wp_usernames.rb @@ -60,7 +60,7 @@ module WpUsernames end def get_nickname_from_url(url) - resp = Browser.instance.get(url, { follow_location: true, max_redirects: 2 }) + resp = Browser.instance.get_and_follow_location(url) nickname = nil if resp.code == 200 nickname = extract_nickname_from_body(resp.body) diff --git a/lib/wpscan/wp_enumerator.rb b/lib/wpscan/wp_enumerator.rb index fa741d34..4dd6b117 100644 --- a/lib/wpscan/wp_enumerator.rb +++ b/lib/wpscan/wp_enumerator.rb @@ -55,7 +55,7 @@ class WpEnumerator targets.each do |target| url = target.get_full_url - request = enum_browser.forge_request(url, { cache_timeout: 0, follow_location: true }) + request = enum_browser.forge_request(url, { cache_ttl: 0, followlocation: true }) request_count += 1 request.on_complete do |response| diff --git a/lib/wpscan/wp_theme.rb b/lib/wpscan/wp_theme.rb index e2bdf160..c3ed7b84 100644 --- a/lib/wpscan/wp_theme.rb +++ b/lib/wpscan/wp_theme.rb @@ -74,7 +74,7 @@ class WpTheme < WpItem # Discover the wordpress theme name by parsing the css link rel def self.find_from_css_link(target_uri) - response = Browser.instance.get(target_uri.to_s, { follow_location: true, max_redirects: 2 }) + response = Browser.instance.get_and_follow_location(target_uri.to_s) # https + domain is optional because of relative links matches = %r{(?:https?://[^"']+)?/([^/]+)/themes/([^"']+)/style.css}i.match(response.body) diff --git a/lib/wpstools/plugins/checker/checker_plugin.rb b/lib/wpstools/plugins/checker/checker_plugin.rb index 974f3039..88af81da 100644 --- a/lib/wpstools/plugins/checker/checker_plugin.rb +++ b/lib/wpstools/plugins/checker/checker_plugin.rb @@ -63,7 +63,7 @@ class CheckerPlugin < Plugin number_of_urls = urls.size urls.each do |url| - request = browser.forge_request(url, { cache_timeout: 0, follow_location: true }) + request = browser.forge_request(url, { cache_ttl: 0, followlocation: true }) request_count += 1 request.on_complete do |response| diff --git a/spec/lib/common/browser_spec.rb b/spec/lib/common/browser_spec.rb index d7c8b18b..58b056ea 100644 --- a/spec/lib/common/browser_spec.rb +++ b/spec/lib/common/browser_spec.rb @@ -354,26 +354,6 @@ describe Browser do #end end - describe '#Browser.generate_cache_key_from_request' do - it '2 requests with the same url, without params must have the same cache_key' do - - url = 'http://example.com' - key1 = Browser.generate_cache_key_from_request(@browser.forge_request(url)) - key2 = Browser.generate_cache_key_from_request(@browser.forge_request(url)) - - key1.should === key2 - end - - it '2 requests with the same url, but with different params should have a different cache_key' do - - url = 'http://example.com' - key1 = Browser.generate_cache_key_from_request(@browser.forge_request(url, params: { login: 'master', password: 'it\'s me !' })) - key2 = Browser.generate_cache_key_from_request(@browser.forge_request(url)) - - key1.should_not == key2 - end - end - describe 'testing caching' do it 'should only do 1 request, and retrieve the other one from the cache' do diff --git a/spec/lib/common/cache_file_store_spec.rb b/spec/lib/common/cache_file_store_spec.rb index 47724faa..c18b93d8 100644 --- a/spec/lib/common/cache_file_store_spec.rb +++ b/spec/lib/common/cache_file_store_spec.rb @@ -61,17 +61,17 @@ describe CacheFileStore do end end - describe '#read_entry (nonexistent entry)' do + describe '#get (nonexistent entry)' do it 'should return nil' do - @cache.read_entry(Digest::SHA1.hexdigest('hello world')).should be_nil + @cache.get(Digest::SHA1.hexdigest('hello world')).should be_nil end end - describe '#write_entry, #read_entry' do + describe '#set, #get' do after :each do - @cache.write_entry(@key, @data, @timeout) - @cache.read_entry(@key).should === @expected + @cache.set(@key, @data, @timeout) + @cache.get(@key).should === @expected end it 'should get the correct entry (string)' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3fb1683f..e8a18644 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -27,7 +27,7 @@ end require File.expand_path(File.dirname(__FILE__) + '/../lib/common/common_helper') -gem 'webmock', '=1.8.11' +#gem 'webmock', '=1.8.11' require 'webmock/rspec' SPEC_DIR = ROOT_DIR + '/spec'