在我之前关于HtmlUnit Skip particular Javascript execution in HTML unit和Fetch Page source using HtmlUnit : URL got stuck的问题
我曾经提到过URL被卡住了。我还发现它因为HtmlUnit库中的一个方法(解析)没有执行而陷入困境。
我做了进一步的工作。如果完成时间超过指定的超时秒数,我会编写代码以退出该方法。
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class HandleHtmlUnitTimeout {
public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException, InterruptedException, TimeoutException
{
Date start = new Date();
String url = "http://ericaweiner.com/collections/";
doWorkWithTimeout(url, 60);
}
public static void doWorkWithTimeout(final String url, long timeoutSecs) throws InterruptedException, TimeoutException {
//maintains a thread for executing the doWork method
ExecutorService executor = Executors.newFixedThreadPool(1);
//logger.info("Starting method with "+timeoutSecs+" seconds as timeout");
//set the executor thread working
final Future<?> future = executor.submit(new Runnable() {
public void run()
{
try
{
getPageSource(url);
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
});
//check the outcome of the executor thread and limit the time allowed for it to complete
try {
future.get(timeoutSecs, TimeUnit.SECONDS);
} catch (Exception e) {
//ExecutionException: deliverer threw exception
//TimeoutException: didn't complete within downloadTimeoutSecs
//InterruptedException: the executor thread was interrupted
//interrupts the worker thread if necessary
future.cancel(true);
//logger.warn("encountered problem while doing some work", e);
throw new TimeoutException();
}finally{
executor.shutdownNow();
}
}
public static void getPageSource(String productPageUrl)
{
try {
if(productPageUrl == null)
{
productPageUrl = "http://ericaweiner.com/collections/";
}
WebClient wb = new WebClient(BrowserVersion.FIREFOX_3_6);
wb.getOptions().setTimeout(120000);
wb.getOptions().setJavaScriptEnabled(true);
wb.getOptions().setThrowExceptionOnScriptError(true);
wb.getOptions().setThrowExceptionOnFailingStatusCode(false);
HtmlPage page = wb.getPage(productPageUrl);
wb.waitForBackgroundJavaScript(4000);
wb.closeAllWindows();
}
catch (FailingHttpStatusCodeException e)
{
e.printStackTrace();
}
catch (MalformedURLException e)
{
e.printStackTrace();
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
这段代码来自doWorkWithTimeout(url,60);方法。但这并没有终止。
当我尝试使用以下代码调用类似的实现时:
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.Logger;
public class HandleScraperTimeOut {
private static Logger logger = Logger.getLogger(HandleScraperTimeOut .class);
public void doWork() throws InterruptedException {
logger.info(new Date()+ "Starting worker method ");
Thread.sleep(20000);
logger.info(new Date()+ "Ending worker method ");
//perform some long running task here...
}
public void doWorkWithTimeout(int timeoutSecs) {
//maintains a thread for executing the doWork method
ExecutorService executor = Executors.newFixedThreadPool(1);
logger.info("Starting method with "+timeoutSecs+" seconds as timeout");
//set the executor thread working
final Future<?> future = executor.submit(new Runnable() {
public void run()
{
try
{
doWork();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
});
//check the outcome of the executor thread and limit the time allowed for it to complete
try {
future.get(timeoutSecs, TimeUnit.SECONDS);
} catch (Exception e) {
//ExecutionException: deliverer threw exception
//TimeoutException: didn't complete within downloadTimeoutSecs
//InterruptedException: the executor thread was interrupted
//interrupts the worker thread if necessary
future.cancel(true);
logger.warn("encountered problem while doing some work", e);
}
executor.shutdown();
}
public static void main(String a[])
{
HandleScraperTimeOut hcto = new HandleScraperTimeOut ();
hcto.doWorkWithTimeout(30);
}
}
如果有人可以看看并告诉我这是什么问题,那将非常有帮助。
有关问题的更多详细信息,您可以查看Skip particular Javascript execution in HTML unit和Fetch Page source using HtmlUnit : URL got stuck
更新1奇怪的是:future.cancel(true);两种情况下都返回TRUE。我的预期是:
更新2它只与http://ericaweiner.com/collections/
URL挂起。如果我给任何其他网址,即http://www.google.com
,http://www.yahoo.com
,它没有提供。在这些情况下,它会抛出IntruptedException并退出Process。
似乎http://ericaweiner.com/collections/
页面源有一些导致问题的元素。
Future.cancel(boolean)返回:
取消意味着线程在取消之前没有完成,取消的标志设置为true,如果请求则线程被中断。
中断线程menans它叫做Thread.interrupt,仅此而已。 Future.cancel(boolean)不检查线程是否实际停止。
所以在这种情况下取消返回true是正确的。
中断线程意味着它应该尽快停止,但不会强制执行。您可以尝试使其停止/失败关闭所需的资源或其他内容。我通常通过从套接字读取(等待传入数据)的线程来做到这一点。我关闭套接字使其停止等待。