Use less memory when brute forcing with a large wordlist

lib/common/collections/wp_users/brute_forcable.rb

@@ -5,17 +5,12 @@ class WpUsers < WpItems
 
     # Brute force each wp_user
     #
-    # To avoid loading the wordlist each time in the wp_user instance
+    # @param [ String ] wordlist The path to the wordlist
-    # It's loaded here, and given to the wp_user
-    #
-    # @param [ String, Array<String> ] wordlist
     # @param [ Hash ] options See WpUser::BruteForcable#brute_force
     #
     # @return [ void ]
     def brute_force(wordlist, options = {})
-      passwords = WpUser::BruteForcable.passwords_from_wordlist(wordlist)
+      self.each { |wp_user| wp_user.brute_force(wordlist, options) }
-
-      self.each { |wp_user| wp_user.brute_force(passwords, options) }
     end
 
   end

lib/common/common_helper.rb

@@ -165,3 +165,11 @@ end
 def get_memory_usage
   `ps -o rss= -p #{Process.pid}`.to_i * 1024 # ps returns the value in KB
 end
+
+# Use the wc system command to count the number of lines in the file
+# instead of using File.open which will to much memory for large file (10 times the size of the file)
+#
+# @return [ Integer ] The number of lines in the given file
+def count_file_lines(file)
+  `wc -l #{file.shellescape}`.split[0].to_i
+end

lib/common/models/wp_user/brute_forcable.rb

@@ -13,7 +13,7 @@ class WpUser < WpItem
     # This means that while we are waiting for browser.max_threads,
     # responses, we are waiting...
     #
-    # @param [ String, Array<String> ] wordlist The wordlist path
+    # @param [ String ] wordlist The wordlist path
     # @param [ Hash ] options
     # @option options [ Boolean ] :verbose
     # @option options [ Boolean ] :show_progression
@@ -23,12 +23,13 @@ class WpUser < WpItem
     def brute_force(wordlist, options = {}, redirect_url = nil)
       browser      = Browser.instance
       hydra        = browser.hydra
-      passwords    = BruteForcable.passwords_from_wordlist(wordlist)
       queue_count  = 0
       found        = false
-      progress_bar = self.progress_bar(passwords.size, options)
+      progress_bar = self.progress_bar(count_file_lines(wordlist), options)
+
+      File.open(wordlist).each do |password|
+        password.chop!
         
-      passwords.each do |password|
         # A successfull login will redirect us to the redirect_to parameter
         # Generate a random one on each request
         unless redirect_url
@@ -123,27 +124,5 @@ class WpUser < WpItem
       valid || false
     end
 
-    # Load the passwords from the wordlist, which can be a file path or
-    # an array or passwords
-    #
-    # @param [ String, Array<String> ] wordlist
-    #
-    # @return [ Array<String> ]
-    def self.passwords_from_wordlist(wordlist)
-      if wordlist.is_a?(String)
-        passwords = []
-
-        File.open(wordlist).each do |line|
-          passwords << line.chop
-        end
-      elsif wordlist.is_a?(Array)
-        passwords = wordlist
-      else
-        raise 'Invalid wordlist, expected String or Array'
-      end
-
-      passwords
-    end
-
   end
 end

lib/environment.rb

@@ -26,6 +26,7 @@ begin
   require 'base64'
   require 'rbconfig'
   require 'pp'
+  require 'shellwords'
   # Third party libs
   require 'typhoeus'
   require 'json'

spec/samples/common/models/wp_user/brute_forcable/wordlist-iso-8859-1.txt

@@ -1,7 +1,7 @@
 password1
 pa55w0rd
-#comment
+#not-a-comment
 admin
 root
- spaceafterandbefore 
 kansei<EFBFBD><EFBFBD>
+ spaceafterandbefore 

spec/samples/common/models/wp_user/brute_forcable/wordlist-utf-8.txt

@@ -1,7 +1,7 @@
 password1
 pa55w0rd
-#comment
+#not-a-coment
 admin
 root
- spaceafterandbefore 
 kansei£Ô
+ spaceafterandbefore 

spec/shared_examples/wp_user/brute_forcable.rb

@@ -4,120 +4,59 @@ shared_examples 'WpUser::BruteForcable' do
   let(:fixtures_dir)  { MODELS_FIXTURES + '/wp_user/brute_forcable' }
   let(:wordlist_iso)  { fixtures_dir + '/wordlist-iso-8859-1.txt' }
   let(:wordlist_utf8) { fixtures_dir + '/wordlist-utf-8.txt' }
+  let(:redirect_url)  { 'http://www.example.com/asdf/' }
   let(:mod)           { WpUser::BruteForcable }
 
   before { Browser.instance.max_threads = 1 }
 
-  describe '::passwords_from_wordlist' do
-    let(:expected)  { %w{password1 pa55w0rd #comment admin root} << ' spaceafterandbefore ' }
-    let(:exception) { 'Invalid wordlist, expected String or Array' }
-
-    after do
-      if @expected
-        mod.passwords_from_wordlist(wordlist).should == @expected
-      else
-        expect { mod.passwords_from_wordlist(wordlist) }.to raise_error(@exception)
-      end
-    end
-
-    context 'when the wordlist is a file' do
-      context 'when encoded with ISO-8859-1' do
-        let(:wordlist) { wordlist_iso }
-
-        it 'returns the expected passwords' do
-          @expected = expected << "kansei\xA3\xD4"
-        end
-      end
-
-      context 'when encoded with UTF-8' do
-        let(:wordlist) { wordlist_utf8 }
-
-        it 'returns the expected passwords' do
-          @expected = expected << 'kansei£Ô'
-        end
-      end
-    end
-
-    context 'when the wordlist is an Array<String>' do
-      let(:wordlist) { %w{hello pwet yolo} }
-
-      it 'returns it' do
-        @expected = wordlist
-      end
-    end
-
-    context 'when the wordlist is invalid' do
-      let(:wordlist) { 200 }
-
-      it 'raises an error' do
-        @exception = exception
-      end
-    end
-  end
-
   describe '#valid_password?' do
     let(:response)     { Typhoeus::Response.new(resp_options) }
     let(:resp_options) { {} }    
-    let(:return_to)    { 'http://www.example.com/asdf/' }
 
     after do
-      wp_user.valid_password?(response, 'password', return_to).should == @expected
+      wp_user.valid_password?(response, 'password', redirect_url).should == @expected
     end
 
     context 'when 302 and valid return_to parameter' do
-      let(:resp_options) { { code: 302, headers: { 'Location' => return_to } } }
+      let(:resp_options) { { code: 302, headers: { 'Location' => redirect_url } } }
 
-      it 'returns true' do
+      it 'returns true' do @expected = true end
-        @expected = true
-      end
     end
 
     context 'when 302 and invalid return_to parameter' do
       let(:resp_options) { { code: 302, headers: { 'Location' => nil } } }
 
-      it 'returns false' do
+      it 'returns false' do @expected = false end
-        @expected = false
-      end
     end
 
     context 'when login_error' do
       let(:resp_options) { { body: '<div id="login_error">' } }
 
-      it 'returns false' do
+      it 'returns false' do @expected = false end
-        @expected = false
-      end
     end
 
     context 'when timeout' do
       let(:resp_options) { { return_code: :operation_timedout } }
 
-      it 'returns false' do
+      it 'returns false' do @expected = false end
-        @expected = false
-      end
     end
 
     context 'when no response from server (status = 0)' do
       let(:resp_options) { { code: 0 } }
 
-      it 'returns false' do
+      it 'returns false' do @expected = false end
-        @expected = false
-      end
     end
 
     context 'when error 50x' do
       let(:resp_options) { { code: 500 } }
 
-      it 'returns false' do
+      it 'returns false' do @expected = false end
-        @expected = false
-      end
     end
 
     context 'when unknown response' do
       let(:resp_options) { { code: 202 } }
 
-      it 'returns false' do
+      it 'returns false' do @expected = false end
-        @expected = false
-      end
     end
   end
 
@@ -127,15 +66,15 @@ shared_examples 'WpUser::BruteForcable' do
   end
 
   describe '#brute_force' do
-    let(:passwords)     { %w{pass1 pass2 yolo kansei£Ô} }
     let(:login) { 'someuser' }
-    let(:redirect_url)  { 'http://www.example.com/asdf/' }
 
     after do
+      [wordlist_utf8, wordlist_iso].each do |wordlist|
         wp_user.login = login
-      wp_user.brute_force(passwords, {}, redirect_url)
+        wp_user.brute_force(wordlist, {}, redirect_url)
         wp_user.password.should == @expected
       end
+    end
 
     context 'when no password is valid' do
       before do
@@ -149,14 +88,29 @@ shared_examples 'WpUser::BruteForcable' do
       end
     end
 
+    context 'when no redirect_url is given' do
+      let(:redirect_url) { nil }
+
+      before do
+        stub_request(:post, wp_user.login_url).to_return(status: 302, headers: { 'Location' => 'wrong-location' } )
+      end
+
+      it 'does not set the @password' do
+        @expected = nil
+      end
+    end
+
     context 'when a password is valid' do
       # Due to the error with .with(body: { log: login }) above
       # We can't use it to stub the request for a specific password
       # So, the first one will be valid
-      before { stub_request(:post, wp_user.login_url).to_return(status: 302, headers: { 'Location' => redirect_url } ) }
+
+      before do
+        stub_request(:post, wp_user.login_url).to_return(status: 302, headers: { 'Location' => redirect_url } )
+      end
 
       it 'sets the @password' do
-        @expected = passwords[0]
+        @expected = 'password1'
       end
     end
   end

spec/shared_examples/wp_users/brute_forcable.rb

@@ -5,19 +5,16 @@ shared_examples 'WpUsers::BruteForcable' do
   describe '#brute_force' do
     let(:range)           { (1..10) }
     let(:wordlist)        { 'somefile.txt'}
-    let(:passwords)       { %w{password 123456 0000} }
     let(:brute_force_opt) { {} }
 
     it 'calls #brute_force on each wp_user' do
       range.each do |id|
         wp_user = WpUser.new(uri, id: id)
-        wp_user.should_receive(:brute_force).with(passwords, brute_force_opt)
+        wp_user.should_receive(:brute_force).with(wordlist, brute_force_opt)
 
         wp_users << wp_user
       end
 
-      WpUser::BruteForcable.stub(:passwords_from_wordlist).with(wordlist).and_return(passwords)
-
       wp_users.brute_force(wordlist, brute_force_opt)
     end
   end