Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Removed time condition on HostBalancer initialization in JUnit test.

luccioman46b5249c20e6
diff --git a/source/net/yacy/crawler/HostBalancer.java b/source/net/yacy/crawler/HostBalancer.java
index 8af3076e6..ab1179b22 100644
--- a/source/net/yacy/crawler/HostBalancer.java
+++ b/source/net/yacy/crawler/HostBalancer.java
@@ -71,10 +71,31 @@ public class HostBalancer implements Balancer {
private final Set<String> roundRobinHostHashes;
private final int onDemandLimit;
+ /**
+ * Create a new instance and asynchronously fills the queue by scanning the hostsPath directory.
+ * @param hostsPath path with persisted hosts queues
+ * @param onDemandLimit
+ * @param exceed134217727
+ */
public HostBalancer(
final File hostsPath,
final int onDemandLimit,
final boolean exceed134217727) {
+ this(hostsPath, onDemandLimit, exceed134217727, true);
+ }
+
+ /**
+ * Create a new instance and fills the queue by scanning the hostsPath directory.
+ * @param hostsPath
+ * @param onDemandLimit
+ * @param exceed134217727
+ * @param asyncInit when true, queue filling from file system is launched asynchronously
+ */
+ public HostBalancer(
+ final File hostsPath,
+ final int onDemandLimit,
+ final boolean exceed134217727,
+ final boolean asyncInit) {
this.hostsPath = hostsPath;
this.onDemandLimit = onDemandLimit;
this.exceed134217727 = exceed134217727;
@@ -83,38 +104,50 @@ public class HostBalancer implements Balancer {
if (!(hostsPath.exists())) hostsPath.mkdirs(); // make the path
this.queues = new ConcurrentHashMap<String, HostQueue>();
this.roundRobinHostHashes = new HashSet<String>();
- init(); // return without wait but starts a thread to fill the queues
+ init(asyncInit); // return without wait but starts a thread to fill the queues
}
/**
- * fills the queue by scanning the hostsPath directory in a thread to
+ * Fills the queue by scanning the hostsPath directory.
+ * @param async when true, launch in a dedicated thread to
* return immediately (as large unfinished crawls may take longer to load)
*/
- private void init() {
- Thread t = new Thread("HostBalancer.init") {
- @Override
- public void run() {
- final String[] hostlist = hostsPath.list();
- for (String hoststr : hostlist) {
- try {
- File queuePath = new File(hostsPath, hoststr);
- HostQueue queue = new HostQueue(queuePath, queues.size() > onDemandLimit, exceed134217727);
- if (queue.isEmpty()) {
- queue.close();
- FileUtils.deletedelete(queuePath);
- } else {
- queues.put(queue.getHostHash(), queue);
- }
- } catch (MalformedURLException | RuntimeException e) {
- log.warn("delete queue due to init error for " + hostsPath.getName() + " host=" + hoststr + " " + e.getLocalizedMessage());
- // if exception thrown we can't init the queue, maybe due to name violation. That won't get better, delete it.
- FileUtils.deletedelete(new File(hostsPath, hoststr));
- }
+ private void init(final boolean async) {
+ if(async) {
+ Thread t = new Thread("HostBalancer.init") {
+ @Override
+ public void run() {
+ runInit();
}
- }
- };
+ };
+
+ t.start();
+ } else {
+ runInit();
+ }
+ }
- t.start();
+ /**
+ * Fills the queue by scanning the hostsPath directory.
+ */
+ private void runInit() {
+ final String[] hostlist = hostsPath.list();
+ for (String hoststr : hostlist) {
+ try {
+ File queuePath = new File(hostsPath, hoststr);
+ HostQueue queue = new HostQueue(queuePath, queues.size() > onDemandLimit, exceed134217727);
+ if (queue.isEmpty()) {
+ queue.close();
+ FileUtils.deletedelete(queuePath);
+ } else {
+ queues.put(queue.getHostHash(), queue);
+ }
+ } catch (MalformedURLException | RuntimeException e) {
+ log.warn("delete queue due to init error for " + hostsPath.getName() + " host=" + hoststr + " " + e.getLocalizedMessage());
+ // if exception thrown we can't init the queue, maybe due to name violation. That won't get better, delete it.
+ FileUtils.deletedelete(new File(hostsPath, hoststr));
+ }
+ }
}
@Override
@@ -196,6 +229,11 @@ public class HostBalancer implements Balancer {
return c;
}
+ /**
+ * @return true when the URL is queued is this or any other HostBalancer
+ * instance (as {@link #depthCache} is shared between all HostBalancer
+ * instances)
+ */
@Override
public boolean has(final byte[] urlhashb) {
if (depthCache.has(urlhashb)) return true;
diff --git a/test/java/net/yacy/crawler/HostBalancerTest.java b/test/java/net/yacy/crawler/HostBalancerTest.java
index a7a5454a0..128eab5a3 100644
--- a/test/java/net/yacy/crawler/HostBalancerTest.java
+++ b/test/java/net/yacy/crawler/HostBalancerTest.java
@@ -8,7 +8,8 @@ import net.yacy.cora.util.SpaceExceededException;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.robots.RobotsTxt;
import net.yacy.data.WorkTables;
-import static net.yacy.kelondro.util.FileUtils.deletedelete;
+import net.yacy.kelondro.util.FileUtils;
+
import org.junit.Test;
import static org.junit.Assert.*;
@@ -16,15 +17,16 @@ public class HostBalancerTest {
final File queuesRoot = new File("test/DATA/INDEX/QUEUES");
final File datadir = new File("test/DATA");
-
+
+ private static final boolean EXCEED_134217727 = true;
+ private static final int ON_DEMAND_LIMIT = 1000;
+
/**
* Test of reopen existing HostBalancer cache to test/demonstrate issue with
* HostQueue for file: protocol
*/
@Test
public void testReopen() throws IOException, SpaceExceededException, InterruptedException {
- boolean exceed134217727 = true;
- int onDemandLimit = 1000;
String hostDir = "C:\\filedirectory";
// prepare one urls for push test
@@ -32,10 +34,9 @@ public class HostBalancerTest {
DigestURL url = new DigestURL(urlstr);
Request req = new Request(url, null);
- deletedelete(queuesRoot); // start clean test
+ FileUtils.deletedelete(queuesRoot); // start clean test
- HostBalancer hb = new HostBalancer(queuesRoot, onDemandLimit, exceed134217727);
- Thread.sleep(100); // wait for file operation
+ HostBalancer hb = new HostBalancer(queuesRoot, ON_DEMAND_LIMIT, EXCEED_134217727, false);
hb.clear();
Thread.sleep(100);
@@ -58,8 +59,7 @@ public class HostBalancerTest {
Thread.sleep(200); // wait a bit for file operation
- hb = new HostBalancer(queuesRoot, onDemandLimit, exceed134217727); // reopen balancer
- Thread.sleep(200); // wait a bit for file operation
+ hb = new HostBalancer(queuesRoot, ON_DEMAND_LIMIT, EXCEED_134217727, false); // reopen balancer
assertEquals("size after reopen (with one existing url)", 1, hb.size()); // expect size=1 from previous push
assertTrue("check existance of pushed url", hb.has(url.hash())); // check url exists (it fails as after reopen internal queue.hosthash is wrong)