fix issue #262

lib/wpscan/web_site/robots_txt.rb

@@ -40,8 +40,12 @@ class WebSite
         end
 
         entries.each do |d|
-          temp = @uri.clone
-          temp.path = d
+          begin
+            temp = @uri.clone
+            temp.path = d
+          rescue URI::Error
+            temp = d
+          end
           return_object << temp.to_s
         end
       end

spec/samples/wpscan/web_site/robots_txt/invalid_robots_2.txt

@@ -0,0 +1,8 @@
+User-agent: *
+Disallow: /wp-admin/
+Disallow: http://10.0.0.0/wp-includes/
+Disallow: wooooza
+Disallow: /ÖÜ()=?
+Allow: /asdf/
+
+Sitemap: http://10.0.0.0/sitemap.xml.gz

spec/shared_examples/web_site/robots_txt.rb

@@ -40,6 +40,16 @@ shared_examples 'WebSite::RobotsTxt' do
         @expected = []
       end
 
+      it 'returns some urls and some strings' do
+        @fixture = fixtures_dir + '/robots_txt/invalid_robots_2.txt'
+        @expected = %w(
+          /ÖÜ()=?
+          http://10.0.0.0/wp-includes/
+          http://example.localhost/asdf/
+          wooooza
+        )
+      end
+
       it 'returns an Array of urls (valid robots.txt)' do
         @fixture = fixtures_dir + '/robots_txt/robots.txt'
         @expected = %w(