From b5d5c4177d65c9fbe3e810c8af8328b7d31c53c6 Mon Sep 17 00:00:00 2001 From: erwanlr Date: Fri, 8 May 2015 09:50:51 +0100 Subject: [PATCH] Removes potential spaces in robots.txt entries - Ref #819 --- lib/wpscan/web_site/robots_txt.rb | 5 ++--- spec/samples/wpscan/web_site/robots_txt/robots.txt | 1 + spec/shared_examples/web_site/robots_txt.rb | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/wpscan/web_site/robots_txt.rb b/lib/wpscan/web_site/robots_txt.rb index f3430403..dfcf22e4 100644 --- a/lib/wpscan/web_site/robots_txt.rb +++ b/lib/wpscan/web_site/robots_txt.rb @@ -15,7 +15,6 @@ class WebSite @uri.clone.merge('robots.txt').to_s end - # Parse robots.txt # @return [ Array ] URLs generated from robots.txt def parse_robots_txt @@ -40,9 +39,9 @@ class WebSite entries.each do |d| begin temp = @uri.clone - temp.path = d + temp.path = d.strip rescue URI::Error - temp = d + temp = d.strip end return_object << temp.to_s end diff --git a/spec/samples/wpscan/web_site/robots_txt/robots.txt b/spec/samples/wpscan/web_site/robots_txt/robots.txt index 39c62f8b..11d9942a 100644 --- a/spec/samples/wpscan/web_site/robots_txt/robots.txt +++ b/spec/samples/wpscan/web_site/robots_txt/robots.txt @@ -5,6 +5,7 @@ Disallow: /wordpress/admin/ Disallow: /wordpress/wp-admin/ Disallow: /wordpress/secret/ Disallow: /Wordpress/wp-admin/ +Disallow: /wp-admin/tralling-space/ Allow: /asdf/ Sitemap: http://10.0.0.0/sitemap.xml.gz diff --git a/spec/shared_examples/web_site/robots_txt.rb b/spec/shared_examples/web_site/robots_txt.rb index 84ceb6cd..6dfce44e 100644 --- a/spec/shared_examples/web_site/robots_txt.rb +++ b/spec/shared_examples/web_site/robots_txt.rb @@ -5,7 +5,7 @@ shared_examples 'WebSite::RobotsTxt' do describe '#robots_url' do it 'returns the correct url' do - expect(web_site.robots_url).to be === 'http://example.localhost/robots.txt' + expect(web_site.robots_url).to eql 'http://example.localhost/robots.txt' end end @@ -57,6 +57,7 @@ shared_examples 'WebSite::RobotsTxt' do http://example.localhost/wordpress/wp-admin/ http://example.localhost/wordpress/secret/ http://example.localhost/Wordpress/wp-admin/ + http://example.localhost/wp-admin/tralling-space/ http://example.localhost/asdf/ ) end @@ -70,6 +71,7 @@ shared_examples 'WebSite::RobotsTxt' do http://example.localhost/wordpress/admin/ http://example.localhost/wordpress/secret/ http://example.localhost/Wordpress/wp-admin/ + http://example.localhost/wp-admin/tralling-space/ http://example.localhost/asdf/ ) stub_request_to_fixture(url: web_site_sub.robots_url, fixture: fixture)