garfield/wpscan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
264bc834b5d1d622b889b718c929e85f364ee32b
wpscan/doc_yard/WebSite/RobotsTxt.html
Christian Mehlmauer 8896f8bf79 docs
2013-08-11 08:58:28 +02:00

516 lines
15 KiB
HTML
Raw Blame History

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>
Module: WebSite::RobotsTxt
&mdash; Documentation by YARD 0.8.5.2
</title>
<link rel="stylesheet" href="../css/style.css" type="text/css" media="screen" charset="utf-8" />
<link rel="stylesheet" href="../css/common.css" type="text/css" media="screen" charset="utf-8" />
<script type="text/javascript" charset="utf-8">
hasFrames = window.top.frames.main ? true : false;
relpath = '../';
framesUrl = "../frames.html#!" + escape(window.location.href);
</script>
<script type="text/javascript" charset="utf-8" src="../js/jquery.js"></script>
<script type="text/javascript" charset="utf-8" src="../js/app.js"></script>
</head>
<body>
<div id="header">
<div id="menu">
<a href="../_index.html">Index (R)</a> &raquo;
<span class='title'><span class='object_link'><a href="../WebSite.html" title="WebSite (class)">WebSite</a></span></span>
&raquo;
<span class="title">RobotsTxt</span>
<div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
</div>
<div id="search">
<a class="full_list_link" id="class_list_link"
href="../class_list.html">
Class List
</a>
<a class="full_list_link" id="method_list_link"
href="../method_list.html">
Method List
</a>
<a class="full_list_link" id="file_list_link"
href="../file_list.html">
File List
</a>
</div>
<div class="clear"></div>
</div>
<iframe id="search_frame"></iframe>
<div id="content"><h1>Module: WebSite::RobotsTxt
</h1>
<dl class="box">
<dt class="r1">Included in:</dt>
<dd class="r1"><span class='object_link'><a href="../WebSite.html" title="WebSite (class)">WebSite</a></span></dd>
<dt class="r2 last">Defined in:</dt>
<dd class="r2 last">lib/wpscan/web_site/robots_txt.rb</dd>
</dl>
<div class="clear"></div>
<h2>
Class Method Summary
<small>(<a href="#" class="summary_toggle">collapse</a>)</small>
</h2>
<ul class="summary">
<li class="protected ">
<span class="summary_signature">
<a href="#known_dirs-class_method" title="known_dirs (class method)">+ (Array) <strong>known_dirs</strong> </a>
</span>
<span class="note title protected">protected</span>
<span class="summary_desc"><div class='inline'></div></span>
</li>
</ul>
<h2>
Instance Method Summary
<small>(<a href="#" class="summary_toggle">collapse</a>)</small>
</h2>
<ul class="summary">
<li class="public ">
<span class="summary_signature">
<a href="#has_robots%3F-instance_method" title="#has_robots? (instance method)">- (Boolean) <strong>has_robots?</strong> </a>
</span>
<span class="summary_desc"><div class='inline'>
<p>Checks if a robots.txt file exists.</p>
</div></span>
</li>
<li class="public ">
<span class="summary_signature">
<a href="#parse_robots_txt-instance_method" title="#parse_robots_txt (instance method)">- (Array) <strong>parse_robots_txt</strong> </a>
</span>
<span class="summary_desc"><div class='inline'>
<p>Parse robots.txt.</p>
</div></span>
</li>
<li class="public ">
<span class="summary_signature">
<a href="#robots_url-instance_method" title="#robots_url (instance method)">- (String) <strong>robots_url</strong> </a>
</span>
<span class="summary_desc"><div class='inline'>
<p>Gets a robots.txt URL.</p>
</div></span>
</li>
</ul>
<div id="class_method_details" class="method_details_list">
<h2>Class Method Details</h2>
<div class="method_details first">
<h3 class="signature first" id="known_dirs-class_method">
+ (<tt><span class='object_link'><a href="../Array.html" title="Array (class)">Array</a></span></tt>) <strong>known_dirs</strong> <span class="extras">(protected)</span>
</h3><div class="docstring">
<div class="discussion">
</div>
</div>
<div class="tags">
<p class="tag_title">Returns:</p>
<ul class="return">
<li>
<span class='type'>(<tt><span class='object_link'><a href="../Array.html" title="Array (class)">Array</a></span></tt>)</span>
</li>
</ul>
</div><table class="source_code">
<tr>
<td>
<pre class="lines">
54
55
56
57
58
59
60
61</pre>
</td>
<td>
<pre class="code"><span class="info file"># File 'lib/wpscan/web_site/robots_txt.rb', line 54</span>
<span class='kw'>def</span> <span class='kw'>self</span><span class='period'>.</span><span class='id identifier rubyid_known_dirs'>known_dirs</span>
<span class='qwords_beg'>%w{
</span><span class='tstring_content'> /</span><span class='words_sep'>
</span><span class='tstring_content'> /wp-admin/</span><span class='words_sep'>
</span><span class='tstring_content'> /wp-includes/</span><span class='words_sep'>
</span><span class='tstring_content'> /wp-content/</span><span class='words_sep'>
</span><span class='tstring_end'> }</span></span>
<span class='kw'>end</span></pre>
</td>
</tr>
</table>
</div>
</div>
<div id="instance_method_details" class="method_details_list">
<h2>Instance Method Details</h2>
<div class="method_details first">
<h3 class="signature first" id="has_robots?-instance_method">
- (<tt>Boolean</tt>) <strong>has_robots?</strong>
</h3><div class="docstring">
<div class="discussion">
<p>Checks if a robots.txt file exists</p>
</div>
</div>
<div class="tags">
<p class="tag_title">Returns:</p>
<ul class="return">
<li>
<span class='type'>(<tt>Boolean</tt>)</span>
</li>
</ul>
</div><table class="source_code">
<tr>
<td>
<pre class="lines">
8
9
10</pre>
</td>
<td>
<pre class="code"><span class="info file"># File 'lib/wpscan/web_site/robots_txt.rb', line 8</span>
<span class='kw'>def</span> <span class='id identifier rubyid_has_robots?'>has_robots?</span>
<span class='const'>Browser</span><span class='period'>.</span><span class='id identifier rubyid_get'>get</span><span class='lparen'>(</span><span class='id identifier rubyid_robots_url'>robots_url</span><span class='rparen'>)</span><span class='period'>.</span><span class='id identifier rubyid_code'>code</span> <span class='op'>==</span> <span class='int'>200</span>
<span class='kw'>end</span></pre>
</td>
</tr>
</table>
</div>
<div class="method_details ">
<h3 class="signature " id="parse_robots_txt-instance_method">
- (<tt><span class='object_link'><a href="../Array.html" title="Array (class)">Array</a></span></tt>) <strong>parse_robots_txt</strong>
</h3><div class="docstring">
<div class="discussion">
<p>Parse robots.txt</p>
</div>
</div>
<div class="tags">
<p class="tag_title">Returns:</p>
<ul class="return">
<li>
<span class='type'>(<tt><span class='object_link'><a href="../Array.html" title="Array (class)">Array</a></span></tt>)</span>
&mdash;
<div class='inline'>
<p>URLs generated from robots.txt</p>
</div>
</li>
</ul>
</div><table class="source_code">
<tr>
<td>
<pre class="lines">
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49</pre>
</td>
<td>
<pre class="code"><span class="info file"># File 'lib/wpscan/web_site/robots_txt.rb', line 23</span>
<span class='kw'>def</span> <span class='id identifier rubyid_parse_robots_txt'>parse_robots_txt</span>
<span class='kw'>return</span> <span class='kw'>unless</span> <span class='id identifier rubyid_has_robots?'>has_robots?</span>
<span class='id identifier rubyid_return_object'>return_object</span> <span class='op'>=</span> <span class='lbracket'>[</span><span class='rbracket'>]</span>
<span class='id identifier rubyid_response'>response</span> <span class='op'>=</span> <span class='const'>Browser</span><span class='period'>.</span><span class='id identifier rubyid_get'>get</span><span class='lparen'>(</span><span class='id identifier rubyid_robots_url'>robots_url</span><span class='period'>.</span><span class='id identifier rubyid_to_s'>to_s</span><span class='rparen'>)</span>
<span class='id identifier rubyid_body'>body</span> <span class='op'>=</span> <span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_body'>body</span>
<span class='comment'># Get all allow and disallow urls
</span> <span class='id identifier rubyid_entries'>entries</span> <span class='op'>=</span> <span class='id identifier rubyid_body'>body</span><span class='period'>.</span><span class='id identifier rubyid_scan'>scan</span><span class='lparen'>(</span><span class='tstring'><span class='regexp_beg'>/</span><span class='tstring_content'>^(?:dis)?allow:\s*(.*)$</span><span class='regexp_end'>/i</span></span><span class='rparen'>)</span>
<span class='kw'>if</span> <span class='id identifier rubyid_entries'>entries</span>
<span class='id identifier rubyid_entries'>entries</span><span class='period'>.</span><span class='id identifier rubyid_flatten!'>flatten!</span>
<span class='id identifier rubyid_entries'>entries</span><span class='period'>.</span><span class='id identifier rubyid_compact'>compact</span><span class='period'>.</span><span class='id identifier rubyid_sort!'>sort!</span>
<span class='id identifier rubyid_wordpress_path'>wordpress_path</span> <span class='op'>=</span> <span class='ivar'>@uri</span><span class='period'>.</span><span class='id identifier rubyid_path'>path</span>
<span class='const'>RobotsTxt</span><span class='period'>.</span><span class='id identifier rubyid_known_dirs'>known_dirs</span><span class='period'>.</span><span class='id identifier rubyid_each'>each</span> <span class='kw'>do</span> <span class='op'>|</span><span class='id identifier rubyid_d'>d</span><span class='op'>|</span>
<span class='id identifier rubyid_entries'>entries</span><span class='period'>.</span><span class='id identifier rubyid_delete'>delete</span><span class='lparen'>(</span><span class='id identifier rubyid_d'>d</span><span class='rparen'>)</span>
<span class='comment'># also delete when wordpress is installed in subdir
</span> <span class='id identifier rubyid_dir_with_subdir'>dir_with_subdir</span> <span class='op'>=</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_wordpress_path'>wordpress_path</span><span class='rbrace'>}</span><span class='tstring_content'>/</span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_d'>d</span><span class='rbrace'>}</span><span class='tstring_end'>&quot;</span></span><span class='period'>.</span><span class='id identifier rubyid_gsub'>gsub</span><span class='lparen'>(</span><span class='tstring'><span class='regexp_beg'>/</span><span class='tstring_content'>\/+</span><span class='regexp_end'>/</span></span><span class='comma'>,</span> <span class='tstring'><span class='tstring_beg'>'</span><span class='tstring_content'>/</span><span class='tstring_end'>'</span></span><span class='rparen'>)</span>
<span class='id identifier rubyid_entries'>entries</span><span class='period'>.</span><span class='id identifier rubyid_delete'>delete</span><span class='lparen'>(</span><span class='id identifier rubyid_dir_with_subdir'>dir_with_subdir</span><span class='rparen'>)</span>
<span class='kw'>end</span>
<span class='id identifier rubyid_entries'>entries</span><span class='period'>.</span><span class='id identifier rubyid_each'>each</span> <span class='kw'>do</span> <span class='op'>|</span><span class='id identifier rubyid_d'>d</span><span class='op'>|</span>
<span class='id identifier rubyid_temp'>temp</span> <span class='op'>=</span> <span class='ivar'>@uri</span><span class='period'>.</span><span class='id identifier rubyid_clone'>clone</span>
<span class='id identifier rubyid_temp'>temp</span><span class='period'>.</span><span class='id identifier rubyid_path'>path</span> <span class='op'>=</span> <span class='id identifier rubyid_d'>d</span>
<span class='id identifier rubyid_return_object'>return_object</span> <span class='op'>&lt;&lt;</span> <span class='id identifier rubyid_temp'>temp</span><span class='period'>.</span><span class='id identifier rubyid_to_s'>to_s</span>
<span class='kw'>end</span>
<span class='kw'>end</span>
<span class='id identifier rubyid_return_object'>return_object</span>
<span class='kw'>end</span></pre>
</td>
</tr>
</table>
</div>
<div class="method_details ">
<h3 class="signature " id="robots_url-instance_method">
- (<tt>String</tt>) <strong>robots_url</strong>
</h3><div class="docstring">
<div class="discussion">
<p>Gets a robots.txt URL</p>
</div>
</div>
<div class="tags">
<p class="tag_title">Returns:</p>
<ul class="return">
<li>
<span class='type'>(<tt>String</tt>)</span>
</li>
</ul>
</div><table class="source_code">
<tr>
<td>
<pre class="lines">
14
15
16
17
18</pre>
</td>
<td>
<pre class="code"><span class="info file"># File 'lib/wpscan/web_site/robots_txt.rb', line 14</span>
<span class='kw'>def</span> <span class='id identifier rubyid_robots_url'>robots_url</span>
<span class='id identifier rubyid_temp'>temp</span> <span class='op'>=</span> <span class='ivar'>@uri</span><span class='period'>.</span><span class='id identifier rubyid_clone'>clone</span>
<span class='id identifier rubyid_temp'>temp</span><span class='period'>.</span><span class='id identifier rubyid_path'>path</span> <span class='op'>=</span> <span class='tstring'><span class='tstring_beg'>'</span><span class='tstring_content'>/robots.txt</span><span class='tstring_end'>'</span></span>
<span class='id identifier rubyid_temp'>temp</span><span class='period'>.</span><span class='id identifier rubyid_to_s'>to_s</span>
<span class='kw'>end</span></pre>
</td>
</tr>
</table>
</div>
</div>
</div>
<div id="footer">
Generated on Sun Aug 11 08:57:55 2013 by
<a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
0.8.5.2 (ruby-1.9.3).
</div>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink