From 8b9aec468a025caaf845604606dba6e1a635a8ad Mon Sep 17 00:00:00 2001 From: erwanlr Date: Wed, 20 Feb 2013 17:34:17 +0100 Subject: [PATCH 01/10] Initial work --- Gemfile | 4 +- lib/common/browser.rb | 55 ++++--------------- lib/common/cache_file_store.rb | 4 +- lib/environment.rb | 1 - lib/wpscan/modules/wp_usernames.rb | 2 +- lib/wpscan/wp_enumerator.rb | 2 +- lib/wpscan/wp_theme.rb | 2 +- .../plugins/checker/checker_plugin.rb | 2 +- spec/lib/common/browser_spec.rb | 20 ------- spec/lib/common/cache_file_store_spec.rb | 10 ++-- spec/spec_helper.rb | 2 +- 11 files changed, 26 insertions(+), 78 deletions(-) diff --git a/Gemfile b/Gemfile index 9092ebd9..5316b28b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,11 +1,11 @@ source "https://rubygems.org" -gem "typhoeus", "0.4.2" +gem "typhoeus", "~>0.6.1" gem "nokogiri" gem "json" group :development, :test do - gem "webmock", "1.8.11" + gem "webmock", "~>1.9.3" gem "simplecov" gem "rspec", :require => "spec" end diff --git a/lib/common/browser.rb b/lib/common/browser.rb index d4d9d17d..2de66412 100644 --- a/lib/common/browser.rb +++ b/lib/common/browser.rb @@ -48,7 +48,7 @@ class Browser @hydra = Typhoeus::Hydra.new( max_concurrency: @max_threads, - timeout: @request_timeout + #connecttimeout: @request_timeout ) # TODO : add an option for the cache dir instead of using a constant @@ -56,8 +56,7 @@ class Browser @cache.clean - # might be in CacheFileStore - setup_cache_handlers + #Typhoeus::Config.cache = @cache end private_class_method :new @@ -146,24 +145,6 @@ class Browser end end - def setup_cache_handlers - @hydra.cache_setter do |request| - @cache.write_entry( - Browser.generate_cache_key_from_request(request), - request.response, - request.cache_timeout - ) - end - - @hydra.cache_getter do |request| - @cache.read_entry( - Browser.generate_cache_key_from_request(request) - ) rescue nil - end - end - - private :setup_cache_handlers - def get(url, params = {}) run_request( forge_request(url, params.merge(method: :get)) @@ -177,10 +158,10 @@ class Browser end def get_and_follow_location(url, params = {}) - params[:max_redirects] ||= 2 + params[:maxredirs] ||= 2 run_request( - forge_request(url, params.merge(method: :get, follow_location: true)) + forge_request(url, params.merge(method: :get, followlocation: true)) ) end @@ -208,12 +189,13 @@ class Browser end end - unless params.has_key?(:disable_ssl_host_verification) - params = params.merge(:disable_ssl_host_verification => true) + # TODO : check if it's the default value into ethon. If so, removed the lines from here + unless params.has_key?(:ssl_verifyhost) + params = params.merge(ssl_verifyhost: 0) end - unless params.has_key?(:disable_ssl_peer_verification) - params = params.merge(:disable_ssl_peer_verification => true) + unless params.has_key?(:ssl_verifypeer) + params = params.merge(ssl_verifypeer: false) end if !params.has_key?(:headers) @@ -223,9 +205,9 @@ class Browser end # Used to enable the cache system if :cache_timeout > 0 - unless params.has_key?(:cache_timeout) - params = params.merge(:cache_timeout => @cache_timeout) - end + #unless params.has_key?(:cache_ttl) + # params = params.merge(cache_ttl: @cache_timeout) + #end params end @@ -247,17 +229,4 @@ class Browser end end end - - # The Typhoeus::Request.cache_key only hash the url :/ - # this one will include the params - # TODO : include also the method (:get, :post, :any) - def self.generate_cache_key_from_request(request) - cache_key = request.cache_key - - if request.params - cache_key = Digest::SHA1.hexdigest("#{cache_key}-#{request.params.hash}") - end - - cache_key - end end diff --git a/lib/common/cache_file_store.rb b/lib/common/cache_file_store.rb index 71bf33c5..e8f0a3cb 100644 --- a/lib/common/cache_file_store.rb +++ b/lib/common/cache_file_store.rb @@ -51,7 +51,7 @@ class CacheFileStore end end - def read_entry(key) + def get(key) entry_file_path = get_entry_file_path(key) if File.exists?(entry_file_path) @@ -59,7 +59,7 @@ class CacheFileStore end end - def write_entry(key, data_to_store, cache_timeout) + def set(key, data_to_store, cache_timeout) if cache_timeout > 0 File.open(get_entry_file_path(key), 'w') do |f| f.write(@serializer.dump(data_to_store)) diff --git a/lib/environment.rb b/lib/environment.rb index 242cc96c..dd9c209f 100644 --- a/lib/environment.rb +++ b/lib/environment.rb @@ -33,7 +33,6 @@ begin require 'rbconfig' require 'pp' # Third party libs - gem 'typhoeus', '=0.4.2' require 'typhoeus' require 'json' require 'nokogiri' diff --git a/lib/wpscan/modules/wp_usernames.rb b/lib/wpscan/modules/wp_usernames.rb index 94f421d7..8ba9b6e8 100644 --- a/lib/wpscan/modules/wp_usernames.rb +++ b/lib/wpscan/modules/wp_usernames.rb @@ -60,7 +60,7 @@ module WpUsernames end def get_nickname_from_url(url) - resp = Browser.instance.get(url, { follow_location: true, max_redirects: 2 }) + resp = Browser.instance.get_and_follow_location(url) nickname = nil if resp.code == 200 nickname = extract_nickname_from_body(resp.body) diff --git a/lib/wpscan/wp_enumerator.rb b/lib/wpscan/wp_enumerator.rb index fa741d34..4dd6b117 100644 --- a/lib/wpscan/wp_enumerator.rb +++ b/lib/wpscan/wp_enumerator.rb @@ -55,7 +55,7 @@ class WpEnumerator targets.each do |target| url = target.get_full_url - request = enum_browser.forge_request(url, { cache_timeout: 0, follow_location: true }) + request = enum_browser.forge_request(url, { cache_ttl: 0, followlocation: true }) request_count += 1 request.on_complete do |response| diff --git a/lib/wpscan/wp_theme.rb b/lib/wpscan/wp_theme.rb index e2bdf160..c3ed7b84 100644 --- a/lib/wpscan/wp_theme.rb +++ b/lib/wpscan/wp_theme.rb @@ -74,7 +74,7 @@ class WpTheme < WpItem # Discover the wordpress theme name by parsing the css link rel def self.find_from_css_link(target_uri) - response = Browser.instance.get(target_uri.to_s, { follow_location: true, max_redirects: 2 }) + response = Browser.instance.get_and_follow_location(target_uri.to_s) # https + domain is optional because of relative links matches = %r{(?:https?://[^"']+)?/([^/]+)/themes/([^"']+)/style.css}i.match(response.body) diff --git a/lib/wpstools/plugins/checker/checker_plugin.rb b/lib/wpstools/plugins/checker/checker_plugin.rb index 974f3039..88af81da 100644 --- a/lib/wpstools/plugins/checker/checker_plugin.rb +++ b/lib/wpstools/plugins/checker/checker_plugin.rb @@ -63,7 +63,7 @@ class CheckerPlugin < Plugin number_of_urls = urls.size urls.each do |url| - request = browser.forge_request(url, { cache_timeout: 0, follow_location: true }) + request = browser.forge_request(url, { cache_ttl: 0, followlocation: true }) request_count += 1 request.on_complete do |response| diff --git a/spec/lib/common/browser_spec.rb b/spec/lib/common/browser_spec.rb index d7c8b18b..58b056ea 100644 --- a/spec/lib/common/browser_spec.rb +++ b/spec/lib/common/browser_spec.rb @@ -354,26 +354,6 @@ describe Browser do #end end - describe '#Browser.generate_cache_key_from_request' do - it '2 requests with the same url, without params must have the same cache_key' do - - url = 'http://example.com' - key1 = Browser.generate_cache_key_from_request(@browser.forge_request(url)) - key2 = Browser.generate_cache_key_from_request(@browser.forge_request(url)) - - key1.should === key2 - end - - it '2 requests with the same url, but with different params should have a different cache_key' do - - url = 'http://example.com' - key1 = Browser.generate_cache_key_from_request(@browser.forge_request(url, params: { login: 'master', password: 'it\'s me !' })) - key2 = Browser.generate_cache_key_from_request(@browser.forge_request(url)) - - key1.should_not == key2 - end - end - describe 'testing caching' do it 'should only do 1 request, and retrieve the other one from the cache' do diff --git a/spec/lib/common/cache_file_store_spec.rb b/spec/lib/common/cache_file_store_spec.rb index 47724faa..c18b93d8 100644 --- a/spec/lib/common/cache_file_store_spec.rb +++ b/spec/lib/common/cache_file_store_spec.rb @@ -61,17 +61,17 @@ describe CacheFileStore do end end - describe '#read_entry (nonexistent entry)' do + describe '#get (nonexistent entry)' do it 'should return nil' do - @cache.read_entry(Digest::SHA1.hexdigest('hello world')).should be_nil + @cache.get(Digest::SHA1.hexdigest('hello world')).should be_nil end end - describe '#write_entry, #read_entry' do + describe '#set, #get' do after :each do - @cache.write_entry(@key, @data, @timeout) - @cache.read_entry(@key).should === @expected + @cache.set(@key, @data, @timeout) + @cache.get(@key).should === @expected end it 'should get the correct entry (string)' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3fb1683f..e8a18644 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -27,7 +27,7 @@ end require File.expand_path(File.dirname(__FILE__) + '/../lib/common/common_helper') -gem 'webmock', '=1.8.11' +#gem 'webmock', '=1.8.11' require 'webmock/rspec' SPEC_DIR = ROOT_DIR + '/spec' From 88e33c5b4ebd9dd73ced19707db01396cb07f7c8 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Thu, 21 Feb 2013 11:49:36 +0100 Subject: [PATCH 02/10] Use Typhoeus master instead of 0.6.1 --- Gemfile | 2 +- lib/environment.rb | 2 ++ lib/wpscan/wp_enumerator.rb | 2 +- spec/spec_helper.rb | 8 ++------ 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index 5316b28b..a6db3c5c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source "https://rubygems.org" -gem "typhoeus", "~>0.6.1" +gem "typhoeus", :git => "https://github.com/typhoeus/typhoeus.git" gem "nokogiri" gem "json" diff --git a/lib/environment.rb b/lib/environment.rb index dd9c209f..a4453599 100644 --- a/lib/environment.rb +++ b/lib/environment.rb @@ -20,6 +20,7 @@ begin # Standard libs require 'rubygems' + require 'bundler/setup' require 'getoptlong' require 'optparse' # Will replace getoptlong require 'uri' @@ -33,6 +34,7 @@ begin require 'rbconfig' require 'pp' # Third party libs + #gem 'typhoeus', :git => "https://github.com/typhoeus/typhoeus.git" require 'typhoeus' require 'json' require 'nokogiri' diff --git a/lib/wpscan/wp_enumerator.rb b/lib/wpscan/wp_enumerator.rb index 4dd6b117..db151b5c 100644 --- a/lib/wpscan/wp_enumerator.rb +++ b/lib/wpscan/wp_enumerator.rb @@ -55,7 +55,7 @@ class WpEnumerator targets.each do |target| url = target.get_full_url - request = enum_browser.forge_request(url, { cache_ttl: 0, followlocation: true }) + request = enum_browser.forge_request(url, cache_ttl: 0, followlocation: true) request_count += 1 request.on_complete do |response| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e8a18644..1dec7eb3 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -20,15 +20,11 @@ # https://github.com/bblimke/webmock # https://github.com/colszowka/simplecov -# Code Coverage (only works with ruby >= 1.9) -if RUBY_VERSION >= '1.9' - require 'simplecov' -end - require File.expand_path(File.dirname(__FILE__) + '/../lib/common/common_helper') -#gem 'webmock', '=1.8.11' require 'webmock/rspec' +# Code Coverage (only works with ruby >= 1.9) +require 'simplecov' if RUBY_VERSION >= '1.9' SPEC_DIR = ROOT_DIR + '/spec' SPEC_LIB_DIR = SPEC_DIR + '/lib' From 4f3225c56ad6c5b3d6742dafc39c636de823d39d Mon Sep 17 00:00:00 2001 From: erwanlr Date: Thu, 21 Feb 2013 12:12:07 +0100 Subject: [PATCH 03/10] CacheFileStore back to the previous state --- lib/common/cache_file_store.rb | 4 +-- spec/lib/common/cache_file_store_spec.rb | 39 +++++++++++------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/common/cache_file_store.rb b/lib/common/cache_file_store.rb index e8f0a3cb..71bf33c5 100644 --- a/lib/common/cache_file_store.rb +++ b/lib/common/cache_file_store.rb @@ -51,7 +51,7 @@ class CacheFileStore end end - def get(key) + def read_entry(key) entry_file_path = get_entry_file_path(key) if File.exists?(entry_file_path) @@ -59,7 +59,7 @@ class CacheFileStore end end - def set(key, data_to_store, cache_timeout) + def write_entry(key, data_to_store, cache_timeout) if cache_timeout > 0 File.open(get_entry_file_path(key), 'w') do |f| f.write(@serializer.dump(data_to_store)) diff --git a/spec/lib/common/cache_file_store_spec.rb b/spec/lib/common/cache_file_store_spec.rb index c18b93d8..7a26f0b8 100644 --- a/spec/lib/common/cache_file_store_spec.rb +++ b/spec/lib/common/cache_file_store_spec.rb @@ -20,15 +20,12 @@ require 'spec_helper' describe CacheFileStore do - - before :all do - @cache_dir = SPEC_CACHE_DIR + '/cache_file_store' - end + let(:cache_dir) { SPEC_CACHE_DIR + '/cache_file_store' } before :each do - Dir.delete(@cache_dir) rescue nil + Dir.delete(cache_dir) rescue nil - @cache = CacheFileStore.new(@cache_dir) + @cache = CacheFileStore.new(cache_dir) end after :each do @@ -37,7 +34,7 @@ describe CacheFileStore do describe '#storage_path' do it 'returns the storage path given in the #new' do - @cache.storage_path.should == @cache_dir + @cache.storage_path.should == cache_dir end end @@ -52,39 +49,39 @@ describe CacheFileStore do it "should remove all files from the cache dir (#{@cache_dir}" do # let's create some files into the directory first (0..5).each do |i| - File.new(@cache_dir + "/file_#{i}.txt", File::CREAT) + File.new(cache_dir + "/file_#{i}.txt", File::CREAT) end - count_files_in_dir(@cache_dir, 'file_*.txt').should == 6 + count_files_in_dir(cache_dir, 'file_*.txt').should == 6 @cache.clean - count_files_in_dir(@cache_dir).should == 0 + count_files_in_dir(cache_dir).should == 0 end end - describe '#get (nonexistent entry)' do + describe '#read_entry (nonexistent entry)' do it 'should return nil' do - @cache.get(Digest::SHA1.hexdigest('hello world')).should be_nil + @cache.read_entry(Digest::SHA1.hexdigest('hello world')).should be_nil end end - describe '#set, #get' do + describe '#write_entry, #read_entry' do after :each do - @cache.set(@key, @data, @timeout) - @cache.get(@key).should === @expected + @cache.write_entry(@key, @data, @timeout) + @cache.read_entry(@key).should === @expected end it 'should get the correct entry (string)' do - @timeout = 10 - @key = 'some_key' - @data = 'Hello World !' + @timeout = 10 + @key = 'some_key' + @data = 'Hello World !' @expected = @data end it 'should not write the entry' do - @timeout = 0 - @key = 'another_key' - @data = 'Another Hello World !' + @timeout = 0 + @key = 'another_key' + @data = 'Another Hello World !' @expected = nil end From b8ee3f84a00cfa9dc8bbd451c6d2a7c092efb795 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Thu, 21 Feb 2013 12:41:43 +0100 Subject: [PATCH 04/10] Typhoeus cache class --- lib/common/browser.rb | 16 +++++++------ lib/common/cache_file_store.rb | 2 +- lib/common/typhoeus_cache.rb | 32 ++++++++++++++++++++++++++ lib/environment.rb | 2 -- spec/lib/common/typhoeus_cache_spec.rb | 3 +++ 5 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 lib/common/typhoeus_cache.rb create mode 100644 spec/lib/common/typhoeus_cache_spec.rb diff --git a/lib/common/browser.rb b/lib/common/browser.rb index 2de66412..0a43b520 100644 --- a/lib/common/browser.rb +++ b/lib/common/browser.rb @@ -17,6 +17,8 @@ # along with this program. If not, see . #++ +require 'common/typhoeus_cache' + class Browser @@instance = nil USER_AGENT_MODES = %w{ static semi-static random } @@ -51,12 +53,12 @@ class Browser #connecttimeout: @request_timeout ) - # TODO : add an option for the cache dir instead of using a constant - @cache = CacheFileStore.new(CACHE_DIR + '/browser') + # TODO : add an argument for the cache dir instead of using a constant + @cache = TyphoeusCache.new(CACHE_DIR + '/browser') @cache.clean - #Typhoeus::Config.cache = @cache + Typhoeus::Config.cache = @cache end private_class_method :new @@ -199,15 +201,15 @@ class Browser end if !params.has_key?(:headers) - params = params.merge(:headers => {'user-agent' => self.user_agent}) + params = params.merge(:headers => {'ser-agent' => self.user_agent}) elsif !params[:headers].has_key?('user-agent') params[:headers]['user-agent'] = self.user_agent end # Used to enable the cache system if :cache_timeout > 0 - #unless params.has_key?(:cache_ttl) - # params = params.merge(cache_ttl: @cache_timeout) - #end + unless params.has_key?(:cache_ttl) + params = params.merge(cache_ttl: @cache_timeout) + end params end diff --git a/lib/common/cache_file_store.rb b/lib/common/cache_file_store.rb index 71bf33c5..5ff42d17 100644 --- a/lib/common/cache_file_store.rb +++ b/lib/common/cache_file_store.rb @@ -68,7 +68,7 @@ class CacheFileStore end def get_entry_file_path(key) - @storage_path + '/' + key + File::join(@storage_path, key) end end diff --git a/lib/common/typhoeus_cache.rb b/lib/common/typhoeus_cache.rb new file mode 100644 index 00000000..811a419a --- /dev/null +++ b/lib/common/typhoeus_cache.rb @@ -0,0 +1,32 @@ +# encoding: UTF-8 +#-- +# WPScan - WordPress Security Scanner +# Copyright (C) 2012-2013 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#++ + +require 'common/cache_file_store' + +class TyphoeusCache < CacheFileStore + + def get(request) + read_entry(request.hash.to_s) + end + + def set(request, response) + write_entry(request.hash.to_s, response, request.cache_ttl) + end + +end diff --git a/lib/environment.rb b/lib/environment.rb index a4453599..ad5939e1 100644 --- a/lib/environment.rb +++ b/lib/environment.rb @@ -34,13 +34,11 @@ begin require 'rbconfig' require 'pp' # Third party libs - #gem 'typhoeus', :git => "https://github.com/typhoeus/typhoeus.git" require 'typhoeus' require 'json' require 'nokogiri' # Custom libs require 'common/browser' - require 'common/cache_file_store' require 'common/custom_option_parser' rescue LoadError => e puts "[ERROR] #{e}" diff --git a/spec/lib/common/typhoeus_cache_spec.rb b/spec/lib/common/typhoeus_cache_spec.rb new file mode 100644 index 00000000..32356a91 --- /dev/null +++ b/spec/lib/common/typhoeus_cache_spec.rb @@ -0,0 +1,3 @@ +# encoding: UTF-8 + +# TODO From 5e971ba3fe6a1f88812b5e40234952dd43504c6d Mon Sep 17 00:00:00 2001 From: erwanlr Date: Thu, 21 Feb 2013 12:43:12 +0100 Subject: [PATCH 05/10] --fail-fast added to rspec --- .rspec | 1 + 1 file changed, 1 insertion(+) diff --git a/.rspec b/.rspec index 4e1e0d2f..ba44b749 100644 --- a/.rspec +++ b/.rspec @@ -1 +1,2 @@ --color +--fail-fast From 79cad5cf55547634c89433ae80896a2d76795408 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Thu, 21 Feb 2013 18:48:48 +0100 Subject: [PATCH 06/10] Ref #53 Typhoeus > 0.4.2 support --- conf/browser.conf.json | 2 +- lib/common/browser.rb | 38 ++++----- lib/common/cache_file_store.rb | 4 +- lib/wpscan/modules/brute_force.rb | 7 +- lib/wpscan/modules/wp_config_backup.rb | 15 +++- lib/wpscan/web_site.rb | 14 ++-- spec/lib/common/browser_spec.rb | 80 +++++++++---------- spec/lib/wpscan/modules/brute_force_spec.rb | 27 ++++--- .../wpscan/modules/wp_config_backup_spec.rb | 9 +-- spec/lib/wpscan/web_site_spec.rb | 12 ++- spec/lib/wpscan/wp_target_spec.rb | 4 +- spec/lib/wpscan/wpscan_helper.rb | 3 +- spec/samples/conf/browser/browser.conf.json | 2 +- .../conf/browser/browser.conf_proxy.json | 2 +- .../conf/browser/browser.conf_proxy_auth.json | 2 +- 15 files changed, 118 insertions(+), 103 deletions(-) diff --git a/conf/browser.conf.json b/conf/browser.conf.json index 7e901d45..21a3821d 100644 --- a/conf/browser.conf.json +++ b/conf/browser.conf.json @@ -14,7 +14,7 @@ //"proxy": "127.0.0.1:3128", //"proxy_auth": "username:password", - "cache_timeout": 600, // 10 minutes, at this time the cache is cleaned before each scan. If this value is set to 0, the cache will be disabled + "cache_ttl": 600, // 10 minutes, at this time the cache is cleaned before each scan. If this value is set to 0, the cache will be disabled "request_timeout": 2000, // 2s diff --git a/lib/common/browser.rb b/lib/common/browser.rb index 0a43b520..86c51b79 100644 --- a/lib/common/browser.rb +++ b/lib/common/browser.rb @@ -30,7 +30,7 @@ class Browser :proxy, :proxy_auth, :max_threads, - :cache_timeout, + :cache_ttl, :request_timeout, :basic_auth ] @@ -113,13 +113,10 @@ class Browser if !auth.include?(:proxy_username) or !auth.include?(:proxy_password) raise_invalid_proxy_format() end - @proxy_auth = auth + @proxy_auth = auth[:proxy_username] + ':' + auth[:proxy_password] elsif auth.is_a?(String) - if matches = %r{([^:]+):(.*)}.match(auth) - @proxy_auth = { - proxy_username: matches[1], - proxy_password: matches[2] - } + if auth.index(':') != nil + @proxy_auth = auth else raise_invalid_proxy_auth_format() end @@ -176,10 +173,10 @@ class Browser def merge_request_params(params = {}) if @proxy - params = params.merge(:proxy => @proxy) + params = params.merge(proxy: @proxy) if @proxy_auth - params = params.merge(@proxy_auth) + params = params.merge(proxyauth: @proxy_auth) end end @@ -191,24 +188,23 @@ class Browser end end - # TODO : check if it's the default value into ethon. If so, removed the lines from here - unless params.has_key?(:ssl_verifyhost) - params = params.merge(ssl_verifyhost: 0) - end + #unless params.has_key?(:ssl_verifyhost) + # params = params.merge(ssl_verifyhost: 0) + #end - unless params.has_key?(:ssl_verifypeer) - params = params.merge(ssl_verifypeer: false) - end + #unless params.has_key?(:ssl_verifypeer) + # params = params.merge(ssl_verifypeer: 0) + #end if !params.has_key?(:headers) - params = params.merge(:headers => {'ser-agent' => self.user_agent}) - elsif !params[:headers].has_key?('user-agent') - params[:headers]['user-agent'] = self.user_agent + params = params.merge(:headers => {'User-Agent' => self.user_agent}) + elsif !params[:headers].has_key?('User-Agent') + params[:headers]['User-Agent'] = self.user_agent end - # Used to enable the cache system if :cache_timeout > 0 + # Used to enable the cache system if :cache_ttl > 0 unless params.has_key?(:cache_ttl) - params = params.merge(cache_ttl: @cache_timeout) + params = params.merge(cache_ttl: @cache_ttl) end params diff --git a/lib/common/cache_file_store.rb b/lib/common/cache_file_store.rb index 5ff42d17..665d5026 100644 --- a/lib/common/cache_file_store.rb +++ b/lib/common/cache_file_store.rb @@ -59,8 +59,8 @@ class CacheFileStore end end - def write_entry(key, data_to_store, cache_timeout) - if cache_timeout > 0 + def write_entry(key, data_to_store, cache_ttl) + if cache_ttl > 0 File.open(get_entry_file_path(key), 'w') do |f| f.write(@serializer.dump(data_to_store)) end diff --git a/lib/wpscan/modules/brute_force.rb b/lib/wpscan/modules/brute_force.rb index 04f560b3..6d43ffcf 100644 --- a/lib/wpscan/modules/brute_force.rb +++ b/lib/wpscan/modules/brute_force.rb @@ -36,10 +36,11 @@ module BruteForce password_found = false File.open(wordlist_path, 'r').each do |password| - # ignore file comments, but will miss passwords if they start with a hash... next if password[0, 1] == '#' + password.strip! + # keep a count of the amount of requests to be sent request_count += 1 queue_count += 1 @@ -52,8 +53,8 @@ module BruteForce request = Browser.instance.forge_request(login_url, { method: :post, - params: { log: URI::encode(username), pwd: URI::encode(password) }, - cache_timeout: 0 + body: { log: URI::encode(username), pwd: URI::encode(password) }, + cache_ttl: 0 } ) diff --git a/lib/wpscan/modules/wp_config_backup.rb b/lib/wpscan/modules/wp_config_backup.rb index 6055e329..e5411370 100644 --- a/lib/wpscan/modules/wp_config_backup.rb +++ b/lib/wpscan/modules/wp_config_backup.rb @@ -23,10 +23,11 @@ module WpConfigBackup # See http://www.feross.org/cmsploit/ # return an array of backup config files url def config_backup - found = [] - backups = WpConfigBackup.config_backup_files - browser = Browser.instance - hydra = browser.hydra + found = [] + backups = WpConfigBackup.config_backup_files + browser = Browser.instance + hydra = browser.hydra + queue_count = 0 backups.each do |file| file_url = @uri.merge(URI.escape(file)).to_s @@ -39,6 +40,12 @@ module WpConfigBackup end hydra.queue(request) + queue_count += 1 + + if queue_count == browser.max_threads + hydra.run + queue_count = 0 + end end hydra.run diff --git a/lib/wpscan/web_site.rb b/lib/wpscan/web_site.rb index d7b8f2fa..97fc88dd 100644 --- a/lib/wpscan/web_site.rb +++ b/lib/wpscan/web_site.rb @@ -48,12 +48,14 @@ class WebSite def xml_rpc_url unless @xmlrpc_url - headers = Browser.instance.get(@uri.to_s).headers_hash - value = headers['x-pingback'] - if value.nil? or value.empty? - @xmlrpc_url = nil - else - @xmlrpc_url = value + headers = Browser.instance.get(@uri.to_s).headers_hash + @xmlrpc_url = nil + + unless headers.nil? + value = headers['X-Pingback'] + unless value.nil? && value.empty? + @xmlrpc_url = value + end end end @xmlrpc_url diff --git a/spec/lib/common/browser_spec.rb b/spec/lib/common/browser_spec.rb index 58b056ea..9b790769 100644 --- a/spec/lib/common/browser_spec.rb +++ b/spec/lib/common/browser_spec.rb @@ -23,7 +23,7 @@ describe Browser do CONFIG_FILE_WITHOUT_PROXY = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf.json' CONFIG_FILE_WITH_PROXY = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf_proxy.json' CONFIG_FILE_WITH_PROXY_AND_AUTH = SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf_proxy_auth.json' - INSTANCE_VARS_TO_CHECK = ['user_agent', 'user_agent_mode', 'available_user_agents', 'proxy', 'max_threads', 'request_timeout', 'cache_timeout'] + INSTANCE_VARS_TO_CHECK = ['user_agent', 'user_agent_mode', 'available_user_agents', 'proxy', 'max_threads', 'request_timeout', 'cache_ttl'] before :all do @json_config_without_proxy = JSON.parse(File.read(CONFIG_FILE_WITHOUT_PROXY)) @@ -31,6 +31,7 @@ describe Browser do end before :each do + Browser::reset @browser = Browser.instance(config_file: CONFIG_FILE_WITHOUT_PROXY) end @@ -100,12 +101,12 @@ describe Browser do it 'should set the correct credentials' do @proxy_auth = { proxy_username: 'user', proxy_password: 'pass' } - @expected = @proxy_auth + @expected = 'user:pass' end it 'should set the correct credentials' do @proxy_auth = 'username:passwd' - @expected = { proxy_username: 'username', proxy_password: 'passwd' } + @expected = @proxy_auth end end @@ -206,10 +207,10 @@ describe Browser do describe '#merge_request_params without proxy' do it 'should return the default params' do expected_params = { - disable_ssl_host_verification: true, - disable_ssl_peer_verification: true, - headers: { 'user-agent' => @browser.user_agent }, - cache_timeout: @json_config_without_proxy['cache_timeout'] + #disable_ssl_host_verification: true, + #disable_ssl_peer_verification: true, + headers: { 'User-Agent' => @browser.user_agent }, + cache_ttl: @json_config_without_proxy['cache_ttl'] } @browser.merge_request_params().should == expected_params @@ -217,25 +218,25 @@ describe Browser do it 'should return the default params with some values overriden' do expected_params = { - disable_ssl_host_verification: false, - disable_ssl_peer_verification: true, - headers: { 'user-agent' => 'Fake IE' }, - cache_timeout: 0 + #disable_ssl_host_verification: false, + #disable_ssl_peer_verification: true, + headers: { 'User-Agent' => 'Fake IE' }, + cache_ttl: 0 } @browser.merge_request_params( - disable_ssl_host_verification: false, - headers: { 'user-agent' => 'Fake IE' }, - cache_timeout: 0 + #disable_ssl_host_verification: false, + headers: { 'User-Agent' => 'Fake IE' }, + cache_ttl: 0 ).should == expected_params end - it 'should return the defaul params with :headers:accept = \'text/html\' (should not override :headers:user-agent)' do + it 'should return the defaul params with :headers:accept = \'text/html\' (should not override :headers:User-Agent)' do expected_params = { - disable_ssl_host_verification: true, - disable_ssl_peer_verification: true, - headers: { 'user-agent' => @browser.user_agent, 'accept' => 'text/html' }, - cache_timeout: @json_config_without_proxy['cache_timeout'] + #disable_ssl_host_verification: true, + #disable_ssl_peer_verification: true, + headers: { 'User-Agent' => @browser.user_agent, 'accept' => 'text/html' }, + cache_ttl: @json_config_without_proxy['cache_ttl'] } @browser.merge_request_params(headers: { 'accept' => 'text/html' }).should == expected_params @@ -244,19 +245,19 @@ describe Browser do it 'should merge the basic-auth' do @browser.basic_auth = 'Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==' expected_params = { - disable_ssl_host_verification: true, - disable_ssl_peer_verification: true, - cache_timeout: @json_config_without_proxy['cache_timeout'], - headers: { + #disable_ssl_host_verification: true, + #disable_ssl_peer_verification: true, + cache_ttl: @json_config_without_proxy['cache_ttl'], + headers: { 'Authorization' => 'Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==', - 'user-agent' => @browser.user_agent + 'User-Agent' => @browser.user_agent } } @browser.merge_request_params().should == expected_params - expected_params[:headers].merge!('user-agent' => 'Fake FF') - @browser.merge_request_params(headers: { 'user-agent' => 'Fake FF' }).should == expected_params + expected_params[:headers].merge!('User-Agent' => 'Fake FF') + @browser.merge_request_params(headers: { 'User-Agent' => 'Fake FF' }).should == expected_params end end @@ -267,10 +268,10 @@ describe Browser do expected_params = { proxy: @json_config_with_proxy['proxy'], - disable_ssl_host_verification: true, - disable_ssl_peer_verification: true, - headers: { 'user-agent' => @json_config_with_proxy['user_agent'] }, - cache_timeout: @json_config_with_proxy['cache_timeout'] + #disable_ssl_host_verification: true, + #disable_ssl_peer_verification: true, + headers: { 'User-Agent' => @json_config_with_proxy['user_agent'] }, + cache_ttl: @json_config_with_proxy['cache_ttl'] } browser.merge_request_params().should == expected_params @@ -282,12 +283,11 @@ describe Browser do expected_params = { proxy: @json_config_with_proxy['proxy'], - proxy_username: 'user', - proxy_password: 'pass', - disable_ssl_host_verification: true, - disable_ssl_peer_verification: true, - headers: { 'user-agent' => @json_config_with_proxy['user_agent'] }, - cache_timeout: @json_config_with_proxy['cache_timeout'] + proxyauth: 'user:pass', + #disable_ssl_host_verification: true, + #disable_ssl_peer_verification: true, + headers: { 'User-Agent' => @json_config_with_proxy['user_agent'] }, + cache_ttl: @json_config_with_proxy['cache_ttl'] } browser.merge_request_params().should == expected_params @@ -300,16 +300,16 @@ describe Browser do end describe '#post' do - it 'should return a Typhoeus::Response wth body = "Welcome Master" if login=master&password=it\'s me !' do + it 'should return a Typhoeus::Response wth body = "Welcome Master" if login=master&password=itsme!' do url = 'http://example.com/' - stub_request(:post, url). - with(body: "login=master&password=it's me !"). + stub_request(:post, url).with(body: { login: 'master', password: 'itsme!' }). to_return(status: 200, body: 'Welcome Master') response = @browser.post( url, - params: { login: 'master', password: 'it\'s me !' } + body: 'login=master&password=itsme!' + #body: { login: 'master', password: 'hello' } # It's should be this line, but it fails ) response.should be_a Typhoeus::Response diff --git a/spec/lib/wpscan/modules/brute_force_spec.rb b/spec/lib/wpscan/modules/brute_force_spec.rb index 1f2b5a55..9470828f 100644 --- a/spec/lib/wpscan/modules/brute_force_spec.rb +++ b/spec/lib/wpscan/modules/brute_force_spec.rb @@ -48,26 +48,29 @@ shared_examples_for 'BruteForce' do passwords << password.strip unless password.strip[0, 1] == '#' end # Last status must be 302 to get full code coverage - passwords.each do |_| - stub_request(:any, @module.login_url).to_return( - { status: 200, body: 'login_error' }, - { status: 0, body: 'no reponse' }, - { status: 50, body: 'server error' }, - { status: 999, body: 'invalid' }, - { status: 302, body: 'FOUND!' } - ) + passwords.each do |password| + stub_request(:post, @module.login_url). + to_return( + { status: 200, body: 'login_error' }, + { status: 0, body: 'no reponse' }, + { status: 500, body: 'server error' }, + { status: 999, body: 'invalid' }, + { status: 302, body: 'FOUND!' } + ) end - user = WpUser.new('admin', 1, nil) + user = WpUser.new('admin', 1, nil) result = @module.brute_force([user], @wordlist) + result.length.should == 1 result.should === [{ name: 'admin', password: 'root' }] end it 'should cover the timeout branch and return an empty array' do - stub_request(:any, @module.login_url).to_timeout - user = WpUser.new('admin', 1, nil) - result = @module.brute_force([user], @wordlist) + stub_request(:post, @module.login_url).to_timeout + + user = WpUser.new('admin', 1, nil) + result = @module.brute_force([user], @wordlist) result.should == [] end end diff --git a/spec/lib/wpscan/modules/wp_config_backup_spec.rb b/spec/lib/wpscan/modules/wp_config_backup_spec.rb index 672e79a6..00d53576 100644 --- a/spec/lib/wpscan/modules/wp_config_backup_spec.rb +++ b/spec/lib/wpscan/modules/wp_config_backup_spec.rb @@ -34,8 +34,7 @@ shared_examples_for 'WpConfigBackup' do @config_backup_files.each do |backup_file| file_url = @module.uri.merge(URI.escape(backup_file)).to_s - stub_request(:get, file_url). - to_return(status: 404, body: '') + stub_request(:get, file_url).to_return(status: 404) end end @@ -50,8 +49,7 @@ shared_examples_for 'WpConfigBackup' do file_url = @module.uri.merge(URI.escape(backup_file)).to_s expected << file_url - stub_request(:get, file_url). - to_return(status: 200, body: File.new(@fixtures_dir + '/wp-config.php')) + stub_request_to_fixture(url: file_url, fixture: @fixtures_dir + '/wp-config.php') end wp_config_backup = @module.config_backup @@ -67,8 +65,7 @@ shared_examples_for 'WpConfigBackup' do file_url = @module.uri.merge(URI.escape(backup_file)).to_s expected << file_url - stub_request(:get, file_url). - to_return(status: 200, body: File.new(@fixtures_dir + '/wp-config.php')) + stub_request_to_fixture(url: file_url, fixture: @fixtures_dir + '/wp-config.php') end wp_config_backup = @module.config_backup diff --git a/spec/lib/wpscan/web_site_spec.rb b/spec/lib/wpscan/web_site_spec.rb index 67df2aca..e4196e27 100644 --- a/spec/lib/wpscan/web_site_spec.rb +++ b/spec/lib/wpscan/web_site_spec.rb @@ -21,6 +21,14 @@ describe 'WebSite' do let(:fixtures_dir) { SPEC_FIXTURES_WPSCAN_WEB_SITE_DIR } subject(:web_site) { WebSite.new('http://example.localhost/') } + before :all do + Browser::reset + Browser.instance( + config_file: SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf.json', + cache_ttl: 0 + ) + end + describe "#new" do its(:url) { should === 'http://example.localhost/' } end @@ -74,7 +82,7 @@ describe 'WebSite' do it 'should return the correct url : http://example.localhost/xmlrpc.php' do xmlrpc = 'http://example.localhost/xmlrpc.php' stub_request(:get, web_site.url). - to_return(status: 200, body: '', headers: { 'X-Pingback' => xmlrpc}) + to_return(status: 200, headers: { 'X-Pingback' => xmlrpc }) web_site.xml_rpc_url.should === xmlrpc end @@ -88,7 +96,7 @@ describe 'WebSite' do describe '#has_xml_rpc?' do it 'should return true' do stub_request(:get, web_site.url). - to_return(status: 200, body: '', headers: { 'X-Pingback' => 'xmlrpc'}) + to_return(status: 200, headers: { 'X-Pingback' => 'xmlrpc' }) web_site.should have_xml_rpc end diff --git a/spec/lib/wpscan/wp_target_spec.rb b/spec/lib/wpscan/wp_target_spec.rb index 45ddf4c4..6d0e141f 100644 --- a/spec/lib/wpscan/wp_target_spec.rb +++ b/spec/lib/wpscan/wp_target_spec.rb @@ -24,11 +24,11 @@ describe WpTarget do let(:target_url) { 'http://example.localhost/' } before :each do - Browser.reset + Browser::reset @options = { config_file: SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf.json', - cache_timeout: 0, + cache_ttl: 0, wp_content_dir: 'wp-content', wp_plugins_dir: 'wp-content/plugins' } diff --git a/spec/lib/wpscan/wpscan_helper.rb b/spec/lib/wpscan/wpscan_helper.rb index c5ea8c63..57349a08 100644 --- a/spec/lib/wpscan/wpscan_helper.rb +++ b/spec/lib/wpscan/wpscan_helper.rb @@ -39,9 +39,10 @@ class WpScanModuleSpec def initialize(target_url) @uri = URI.parse(add_trailing_slash(add_http_protocol(target_url))) + Browser::reset Browser.instance( config_file: SPEC_FIXTURES_CONF_DIR + '/browser/browser.conf.json', - cache_timeout: 0 + cache_ttl: 0 ) end diff --git a/spec/samples/conf/browser/browser.conf.json b/spec/samples/conf/browser/browser.conf.json index 356178c9..21464e08 100644 --- a/spec/samples/conf/browser/browser.conf.json +++ b/spec/samples/conf/browser/browser.conf.json @@ -1,7 +1,7 @@ { "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0) Gecko/20100101 Firefox/9.0", "user_agent_mode": "static", - "cache_timeout": 300, + "cache_ttl": 300, "request_timeout": 2000, "max_threads": 5 } diff --git a/spec/samples/conf/browser/browser.conf_proxy.json b/spec/samples/conf/browser/browser.conf_proxy.json index 37038c65..175569f4 100644 --- a/spec/samples/conf/browser/browser.conf_proxy.json +++ b/spec/samples/conf/browser/browser.conf_proxy.json @@ -2,6 +2,6 @@ "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:10.0) Gecko/20100101 Firefox/11.0", "user_agent_mode": "static", "proxy": "127.0.0.1:3038", - "cache_timeout": 300, + "cache_ttl": 300, "request_timeout": 2000 } diff --git a/spec/samples/conf/browser/browser.conf_proxy_auth.json b/spec/samples/conf/browser/browser.conf_proxy_auth.json index 75e10fdc..b5fea1fd 100644 --- a/spec/samples/conf/browser/browser.conf_proxy_auth.json +++ b/spec/samples/conf/browser/browser.conf_proxy_auth.json @@ -3,6 +3,6 @@ "user_agent_mode": "static", "proxy": "127.0.0.1:3038", "proxy_auth": "user:pass", - "cache_timeout": 300, + "cache_ttl": 300, "request_timeout": 2000 } From a71765034ba7ead6beb509c32f71091bc34ddc09 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Fri, 1 Mar 2013 15:20:53 +0100 Subject: [PATCH 07/10] Request#cache_key implementation --- lib/common/typhoeus_cache.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/common/typhoeus_cache.rb b/lib/common/typhoeus_cache.rb index 811a419a..de21de8d 100644 --- a/lib/common/typhoeus_cache.rb +++ b/lib/common/typhoeus_cache.rb @@ -19,14 +19,26 @@ require 'common/cache_file_store' +# Implementaion of a cache_key (Request:hash cannot be used as such) +# See https://github.com/typhoeus/typhoeus/issues/277 +module Typhoeus + class Request + module Cacheable + def cache_key + Digest::SHA2.hexdigest("#{url}-#{options[:body]}-#{options[:method]}") + end + end + end +end + class TyphoeusCache < CacheFileStore def get(request) - read_entry(request.hash.to_s) + read_entry(request.cache_key) end def set(request, response) - write_entry(request.hash.to_s, response, request.cache_ttl) + write_entry(request.cache_key, response, request.cache_ttl) end end From a4b3c300997f3ee39bd5ddc3da19d75df494bcd5 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Mon, 4 Mar 2013 11:00:18 +0100 Subject: [PATCH 08/10] Uses Typhoeus 0.6.2 instead of master --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index a6db3c5c..51f013d9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source "https://rubygems.org" -gem "typhoeus", :git => "https://github.com/typhoeus/typhoeus.git" +gem "typhoeus", "~>0.6.2" gem "nokogiri" gem "json" From ae76db5238494f0a81b63873f759630378b08631 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Mon, 4 Mar 2013 11:17:40 +0100 Subject: [PATCH 09/10] Typhoeus::Request#cache_key comment updated --- lib/common/typhoeus_cache.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/common/typhoeus_cache.rb b/lib/common/typhoeus_cache.rb index de21de8d..14f65a0a 100644 --- a/lib/common/typhoeus_cache.rb +++ b/lib/common/typhoeus_cache.rb @@ -19,8 +19,7 @@ require 'common/cache_file_store' -# Implementaion of a cache_key (Request:hash cannot be used as such) -# See https://github.com/typhoeus/typhoeus/issues/277 +# Implementaion of a cache_key (Typhoeus::Request#hash has too many options) module Typhoeus class Request module Cacheable From 2453ad7ace3f87154638fbb852d3441128199ba1 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Mon, 4 Mar 2013 11:20:20 +0100 Subject: [PATCH 10/10] cache_key reduced to 32 chars --- lib/common/typhoeus_cache.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/typhoeus_cache.rb b/lib/common/typhoeus_cache.rb index 14f65a0a..a3039c26 100644 --- a/lib/common/typhoeus_cache.rb +++ b/lib/common/typhoeus_cache.rb @@ -24,7 +24,7 @@ module Typhoeus class Request module Cacheable def cache_key - Digest::SHA2.hexdigest("#{url}-#{options[:body]}-#{options[:method]}") + Digest::SHA2.hexdigest("#{url}-#{options[:body]}-#{options[:method]}")[0..32] end end end