From c4c8a8b7034cb2fa88e747b2a6a476979439bbef Mon Sep 17 00:00:00 2001
From: erwanlr <erwan.lr@gmail.com>
Date: Sat, 29 Mar 2014 12:16:23 +0100
Subject: [PATCH] Memory improvements during wp-items enumerations

---
 lib/common/collections/wp_items/detectable.rb | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/common/collections/wp_items/detectable.rb b/lib/common/collections/wp_items/detectable.rb
index 2f4309ea..c0497eaa 100755
--- a/lib/common/collections/wp_items/detectable.rb
+++ b/lib/common/collections/wp_items/detectable.rb
@@ -17,6 +17,7 @@ class WpItems < Array
       hydra            = browser.hydra
       targets          = targets_items(wp_target, options)
       progress_bar     = progress_bar(targets.size, options)
+      queue_count      = 0
       exist_options    = {
         error_404_hash:  wp_target.error_404_hash,
         homepage_hash:   wp_target.homepage_hash,
@@ -43,8 +44,16 @@ class WpItems < Array
         end
 
         hydra.queue(request)
+        queue_count += 1
+
+        if queue_count >= browser.max_threads
+          hydra.run
+          queue_count = 0
+          puts "Sent #{browser.max_threads} requests ..." if options[:verbose]
+        end
       end
 
+      # run the remaining requests
       hydra.run
       results.sort!
       results # can't just return results.sort because the #sort returns an array, and we want a WpItems