Detection of the wordlist charset

lib/common/hacks.rb

@@ -70,3 +70,12 @@ def puts(o = '')
   end
   super(o)
 end
+
+class File
+  # @param [ String ] file_path
+  #
+  # @return [ String ] The charset of the file
+  def self.charset(file_path)
+    %x{file -i #{file_path}}[%r{charset=([^\n]+)\n}, 1]
+  end
+end

lib/common/models/wp_user/brute_forcable.rb

@@ -9,14 +9,15 @@ class WpUser < WpItem
     # @return [ void ]
     def brute_force(wordlist, options = {})
       hydra               = Browser.instance.hydra
+      wordlist_charset    = File.charset(wordlist)
       number_of_passwords = BruteForcable.lines_in_file(wordlist)
       login_url           = @uri.merge('wp-login.php').to_s
 
-      queue_count    = 0
-      request_count  = 0
+      queue_count         = 0
+      request_count       = 0
 
-      File.open(wordlist, 'r').each do |line|
-        line.strip!
+      File.open(wordlist, "r:#{wordlist_charset}").each do |line|
+        line.encode!('UTF-8').strip!
         # ignore file comments, but will miss passwords if they start with a hash...
         next if line[0, 1] == '#'
 
@@ -101,7 +102,7 @@ class WpUser < WpItem
     # @return [ Integer ]
     def self.lines_in_file(file_path)
       lines = 0
-      File.open(file_path, 'r').each do |line|
+      File.open(file_path, 'rb').each do |line|
         lines += 1 if line.strip[0,1] != '#'
       end
       lines

spec/samples/common/models/wp_user/brute_forcable/wordlist-iso-8859-1.txt

@@ -0,0 +1,6 @@
+password1
+pa55w0rd
+#comment
+admin
+root
+kansei<EFBFBD><EFBFBD>

spec/samples/common/models/wp_user/brute_forcable/wordlist-utf-8.txt

@@ -1,6 +1,6 @@
 password1
 pa55w0rd
-# comment
+#comment
 admin
 root
 kansei£Ô

spec/shared_examples/wp_user/brute_forcable.rb

@@ -2,7 +2,7 @@
 
 shared_examples 'WpUser::BruteForcable' do
   let(:fixtures_dir) { MODELS_FIXTURES + '/wp_user/brute_forcable' }
-  let(:wordlist)     { fixtures_dir + '/wordlist.txt' }
+  let(:wordlist)     { fixtures_dir + '/wordlist-iso-8859-1.txt' }
   let(:mod)          { WpUser::BruteForcable }
   let(:login_url)    { uri.merge('wp-login.php').to_s }
 
@@ -72,11 +72,31 @@ shared_examples 'WpUser::BruteForcable' do
     end
   end
 
+  describe 'wordlist charset' do
+    let(:expected) { %w{password1 pa55w0rd #comment admin root kansei£Ô} }
+
+    %w{wordlist-iso-8859-1.txt wordlist-utf-8.txt}.each do |file|
+      it 'contains the expected lines' do
+        file    = fixtures_dir + '/' + file
+        charset = File.charset(file)
+
+        lines = []
+        File.open(file, "r:#{charset}").each do |line|
+          lines << line.encode!('UTF-8').strip!
+        end
+
+        lines.should == expected
+      end
+    end
+  end
+
   describe '#brute_force' do
     let(:passwords) {
       passwords = []
-      File.open(wordlist, 'r').each do |line|
-        line.strip!
+      charset   = File.charset(wordlist)
+
+      File.open(wordlist, "r:#{charset}").each do |line|
+        line.encode!('UTF-8').strip!
         passwords << line unless line[0,1] == '#'
       end
       passwords