Removes potential spaces in robots.txt entries - Ref #819

lib/wpscan/web_site/robots_txt.rb

@@ -15,7 +15,6 @@ class WebSite
       @uri.clone.merge('robots.txt').to_s
     end
 
-
     # Parse robots.txt
     # @return [ Array ] URLs generated from robots.txt
     def parse_robots_txt
@@ -40,9 +39,9 @@ class WebSite
         entries.each do |d|
           begin
             temp = @uri.clone
-            temp.path = d
+            temp.path = d.strip
           rescue URI::Error
-            temp = d
+            temp = d.strip
           end
           return_object << temp.to_s
         end

spec/samples/wpscan/web_site/robots_txt/robots.txt

@@ -5,6 +5,7 @@ Disallow: /wordpress/admin/
 Disallow: /wordpress/wp-admin/
 Disallow: /wordpress/secret/
 Disallow: /Wordpress/wp-admin/
+Disallow: /wp-admin/tralling-space/ 
 Allow: /asdf/
 
 Sitemap: http://10.0.0.0/sitemap.xml.gz

spec/shared_examples/web_site/robots_txt.rb

@@ -5,7 +5,7 @@ shared_examples 'WebSite::RobotsTxt' do
 
   describe '#robots_url' do
     it 'returns the correct url' do
-      expect(web_site.robots_url).to be === 'http://example.localhost/robots.txt'
+      expect(web_site.robots_url).to eql 'http://example.localhost/robots.txt'
     end
   end
 
@@ -57,6 +57,7 @@ shared_examples 'WebSite::RobotsTxt' do
           http://example.localhost/wordpress/wp-admin/
           http://example.localhost/wordpress/secret/
           http://example.localhost/Wordpress/wp-admin/
+          http://example.localhost/wp-admin/tralling-space/
           http://example.localhost/asdf/
         )
       end
@@ -70,6 +71,7 @@ shared_examples 'WebSite::RobotsTxt' do
             http://example.localhost/wordpress/admin/
             http://example.localhost/wordpress/secret/
             http://example.localhost/Wordpress/wp-admin/
+            http://example.localhost/wp-admin/tralling-space/
             http://example.localhost/asdf/
           )
         stub_request_to_fixture(url: web_site_sub.robots_url, fixture: fixture)