Removes potential spaces in robots.txt entries - Ref #819

spec/samples/wpscan/web_site/robots_txt/robots.txt

@@ -5,6 +5,7 @@ Disallow: /wordpress/admin/
 Disallow: /wordpress/wp-admin/
 Disallow: /wordpress/secret/
 Disallow: /Wordpress/wp-admin/
+Disallow: /wp-admin/tralling-space/ 
 Allow: /asdf/
 
 Sitemap: http://10.0.0.0/sitemap.xml.gz

spec/shared_examples/web_site/robots_txt.rb

@@ -5,7 +5,7 @@ shared_examples 'WebSite::RobotsTxt' do
 
   describe '#robots_url' do
     it 'returns the correct url' do
-      expect(web_site.robots_url).to be === 'http://example.localhost/robots.txt'
+      expect(web_site.robots_url).to eql 'http://example.localhost/robots.txt'
     end
   end
 
@@ -57,6 +57,7 @@ shared_examples 'WebSite::RobotsTxt' do
           http://example.localhost/wordpress/wp-admin/
           http://example.localhost/wordpress/secret/
           http://example.localhost/Wordpress/wp-admin/
+          http://example.localhost/wp-admin/tralling-space/
           http://example.localhost/asdf/
         )
       end
@@ -70,6 +71,7 @@ shared_examples 'WebSite::RobotsTxt' do
             http://example.localhost/wordpress/admin/
             http://example.localhost/wordpress/secret/
             http://example.localhost/Wordpress/wp-admin/
+            http://example.localhost/wp-admin/tralling-space/
             http://example.localhost/asdf/
           )
         stub_request_to_fixture(url: web_site_sub.robots_url, fixture: fixture)