Fixes the location of the robots.txt check
This commit is contained in:
@@ -12,9 +12,7 @@ class WebSite
|
||||
# Gets a robots.txt URL
|
||||
# @return [ String ]
|
||||
def robots_url
|
||||
temp = @uri.clone
|
||||
temp.path = '/robots.txt'
|
||||
temp.to_s
|
||||
@uri.clone.merge('robots.txt').to_s
|
||||
end
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user