Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Copy image resources contained in donation iframe.

luccioman46b883654893
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java
index 4e26f6dc4..4dcbb98f3 100644
--- a/source/net/yacy/yacy.java
+++ b/source/net/yacy/yacy.java
@@ -39,13 +39,22 @@ import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.Properties;
import java.util.concurrent.Semaphore;
+
+import com.google.common.io.Files;
+
import net.yacy.cora.date.GenericFormatter;
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.federate.yacy.CacheStrategy;
+import net.yacy.cora.order.Digest;
import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.sorting.Array;
import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.crawler.retrieval.Response;
+import net.yacy.document.Document;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
import net.yacy.http.Jetty9HttpServerImpl;
@@ -54,17 +63,11 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.OS;
+import net.yacy.peers.Seed;
import net.yacy.peers.operation.yacyBuildProperties;
import net.yacy.peers.operation.yacyRelease;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
-import com.google.common.io.Files;
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.federate.yacy.CacheStrategy;
-import net.yacy.cora.order.Digest;
-import net.yacy.cora.protocol.ConnectionInfo;
-import net.yacy.crawler.retrieval.Response;
-import net.yacy.peers.Seed;
import net.yacy.server.serverSwitch;
import net.yacy.utils.translation.TranslatorXliff;
@@ -265,22 +268,7 @@ public final class yacy {
//final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
// copy the donate iframe (better to copy this once here instead of doing this in an actual iframe in the search result)
- final File wwwEnvPath = new File(htDocsPath, "env");
- mkdirIfNeseccary(wwwEnvPath);
- final String iframesource = sb.getConfig("donation.iframesource", "");
- final String iframetarget = sb.getConfig("donation.iframetarget", "");
- final File iframefile = new File(htDocsPath, iframetarget);
- if (!iframefile.exists()) new Thread() {
- @Override
- public void run() {
- final ClientIdentification.Agent agent = ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName);
- Response response;
- try {
- response = sb.loader == null ? null : sb.loader.load(sb.loader.request(new DigestURL(iframesource), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
- if (response != null) FileUtils.copy(response.getContent(), iframefile);
- } catch (Throwable e) {}
- }
- }.start();
+ importDonationIFrame(sb, htDocsPath);
// create default notifier picture
File notifierFile = new File(htDocsPath, "notifier.gif");
@@ -421,6 +409,59 @@ public final class yacy {
} catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
}
+ /**
+ * Concurrently import the donation iframe content to serve it directly from this peer.
+ * @param switchBoard the SwitchBoard instance. Must not be null.
+ * @param htDocsDirectory the custom htdocs directory. Must not be null.
+ */
+ private static void importDonationIFrame(final Switchboard switchBoard, final File htDocsDirectory) {
+ final File wwwEnvPath = new File(htDocsDirectory, "env");
+ mkdirIfNeseccary(wwwEnvPath);
+ final String iframesource = switchBoard.getConfig("donation.iframesource", "");
+ final String iframetarget = switchBoard.getConfig("donation.iframetarget", "");
+ final File iframefile = new File(htDocsDirectory, iframetarget);
+ if (!iframefile.exists()) new Thread() {
+ @Override
+ public void run() {
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName);
+ Response documentResponse;
+ try {
+ /* Load the donation html frame content */
+ documentResponse = switchBoard.loader == null ? null : switchBoard.loader.load(switchBoard.loader.request(new DigestURL(iframesource), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
+ if (documentResponse != null) {
+ Document[] documents = documentResponse.parse();
+ if(documents != null && documents.length > 0 && documents[0] != null) {
+ Document donateDocument = documents[0];
+ String donateDocContent = new String(documentResponse.getContent(), donateDocument.getCharset());
+ /* Load image resources contained in the page */
+ if(donateDocument.getImages() != null) {
+ for(DigestURL imgURL : donateDocument.getImages().keySet()) {
+ try {
+ Response response = switchBoard.loader.load(switchBoard.loader.request(imgURL, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
+ if (response != null) {
+ String imgFileName = imgURL.getFileName();
+ /* Store each image in the same directory as the iframe target file */
+ FileUtils.copy(response.getContent(), new File(iframefile.getParentFile(), imgFileName));
+
+ /* Transform the original image URL to a relative one */
+ donateDocContent = donateDocContent.replace(imgURL.getURL().toString(), imgFileName);
+ }
+ } catch(IOException e) {
+ /* Failing to load one image should not stop the whole task */
+ ConcurrentLog.warn("STARTUP", "Donation frame retrieval : could not get an image resource.", e);
+ }
+ }
+ }
+ FileUtils.copy(donateDocContent.getBytes(donateDocument.getCharset()), iframefile);
+ }
+ }
+ } catch (Exception e) {
+ ConcurrentLog.warn("STARTUP", "Could not retrieve donation frame content.", e);
+ }
+ }
+ }.start();
+ }
+
/**
* @param f
*/