Bugfixing

lib/wpstools/generate_list.rb

@@ -63,7 +63,7 @@ class Generate_List
     page_count = 1
     queue_count = 0
 
-    (1...pages.to_i).each do |page|
+    (1...(pages.to_i+1)).each do |page|
       # First page has another URL
       url = (page == 1) ? @popular_url : @popular_url + 'page/' + page.to_s + '/'
       request = @browser.forge_request(url)
@@ -75,7 +75,7 @@ class Generate_List
         page_count += 1
         response.body.scan(@popular_regex).each do |item|
           puts "[+] Found popular #{@type}: #{item}" if @verbose
-          found_items << item
+          found_items << item[0]
         end
       end
 
@@ -90,15 +90,17 @@ class Generate_List
 
     @hydra.run
 
-    found_items.uniq
-    found_items.sort
+    found_items.sort!
+    found_items.uniq!
     return found_items
   end
 
   # Save the file
   def save(items)
-    puts "[*] We have parsed #{items} #{@type}s"
-    File.open(@file_name, 'w') { |f| f.write(items) }
+    items.sort!
+    items.uniq!
+    puts "[*] We have parsed #{items.length} #{@type}s"
+    File.open(@file_name, 'w') { |f| f.puts(items) }
     puts "New #{@file_name} file created"
   end
 

lib/wpstools/parse_svn.rb

@@ -30,7 +30,7 @@ class Svn_Parser
     @svn_hydra = @svn_browser.hydra
   end
   
-  def parse(dirs = nil)
+  def parse(dirs=nil)
     if dirs == nil
       dirs = get_root_directories
     end
@@ -43,11 +43,11 @@ class Svn_Parser
   def get_root_directories
     dirs = []
     rootindex = @svn_browser.get(@svn_root).body
-    rootindex.scan(%r{<li><a href=".*">(.*)/</a></li>}i).each do |dir|
+    rootindex.scan(%r{<li><a href=".+">(.+)/</a></li>}i).each do |dir|
       dirs << dir[0]
     end
-    dirs.uniq
-    dirs.sort
+    dirs.sort!
+    dirs.uniq!
     return dirs
   end
 
@@ -62,14 +62,14 @@ class Svn_Parser
         # trunk folder present
         if contains_trunk(response)
           puts "[+] Adding trunk on #{dir}" if @verbose
-          urls << (svnurl << "trunk/")
+          urls << { :name => dir, :folder => "trunk"}
         # no trunk folder. This is true on theme svn repos
         else
-          folders = response.body.scan(%r{^\s*<li><a href="(.*)/">.*/</a></li>$}i)
+          folders = response.body.scan(%r{^\s*<li><a href="(.+)/">.+/</a></li>$}i)
           if folders != nil and folders.length > 0
             last_version = folders.last[0]
             puts "[+] Adding #{last_version} on #{dir}" if @verbose
-            urls << (svnurl + last_version + "/")
+            urls << { :name => dir, :folder => last_version}
           else
             puts "[+] No content in #{dir}" if @verbose
           end
@@ -89,20 +89,24 @@ class Svn_Parser
   end
   
   # Get a file in each directory
-  def get_svn_file_entries(urls)
+  def get_svn_file_entries(dirs)
     entries = []
     queue_count = 0
-    urls.each do |url|
+    dirs.each do |dir|
+      url = @svn_root + dir[:name] + "/" + dir[:folder] + "/"
       request = @svn_browser.forge_request(url)
       request.on_complete do |response|
         puts "[+] Parsing url #{url} [#{response.code.to_s}]" if @verbose
-        file = response.body[%r{<li><a href="(.*\..*)">.*</a></li>}i, 1]
+        file = response.body[%r{<li><a href="(.+\.[^/]+)">.+</a></li>}i, 1]
         # TODO: recursive parsing of subdirectories if there is no file in the root directory
+        path = dir[:name] + "/"
         if file
-          url += "/" + file
-          entries << url
+          path += file
+          entries << path
+          puts "[+] Added #{path}" if @verbose
         elsif @keep_empty_dirs
-          entries << url
+          entries << path
+          puts "[+] Added #{path}" if @verbose
         end
       end
       queue_count += 1