From d107613e402185c3626807605cb2cc1218497b5e Mon Sep 17 00:00:00 2001 From: Christian Mehlmauer Date: Mon, 12 Aug 2013 22:01:24 +0200 Subject: [PATCH] fix issue #262 --- lib/wpscan/web_site/robots_txt.rb | 8 ++++++-- .../wpscan/web_site/robots_txt/invalid_robots_2.txt | 8 ++++++++ spec/shared_examples/web_site/robots_txt.rb | 10 ++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 spec/samples/wpscan/web_site/robots_txt/invalid_robots_2.txt diff --git a/lib/wpscan/web_site/robots_txt.rb b/lib/wpscan/web_site/robots_txt.rb index 00d54618..714667a3 100644 --- a/lib/wpscan/web_site/robots_txt.rb +++ b/lib/wpscan/web_site/robots_txt.rb @@ -40,8 +40,12 @@ class WebSite end entries.each do |d| - temp = @uri.clone - temp.path = d + begin + temp = @uri.clone + temp.path = d + rescue URI::Error + temp = d + end return_object << temp.to_s end end diff --git a/spec/samples/wpscan/web_site/robots_txt/invalid_robots_2.txt b/spec/samples/wpscan/web_site/robots_txt/invalid_robots_2.txt new file mode 100644 index 00000000..19fa8dc5 --- /dev/null +++ b/spec/samples/wpscan/web_site/robots_txt/invalid_robots_2.txt @@ -0,0 +1,8 @@ +User-agent: * +Disallow: /wp-admin/ +Disallow: http://10.0.0.0/wp-includes/ +Disallow: wooooza +Disallow: /ÖÜ()=? +Allow: /asdf/ + +Sitemap: http://10.0.0.0/sitemap.xml.gz diff --git a/spec/shared_examples/web_site/robots_txt.rb b/spec/shared_examples/web_site/robots_txt.rb index 2264196f..abef2f54 100644 --- a/spec/shared_examples/web_site/robots_txt.rb +++ b/spec/shared_examples/web_site/robots_txt.rb @@ -40,6 +40,16 @@ shared_examples 'WebSite::RobotsTxt' do @expected = [] end + it 'returns some urls and some strings' do + @fixture = fixtures_dir + '/robots_txt/invalid_robots_2.txt' + @expected = %w( + /ÖÜ()=? + http://10.0.0.0/wp-includes/ + http://example.localhost/asdf/ + wooooza + ) + end + it 'returns an Array of urls (valid robots.txt)' do @fixture = fixtures_dir + '/robots_txt/robots.txt' @expected = %w(