我需要在类库中创建一个方法来获取URL的内容(可以通过JavaScript动态填充)。
我一无所知,但谷歌搜索了一整天这是我想出的:(大部分代码来自here)
using System;
using System.Threading.Tasks;
using System.Threading;
using System.Windows.Forms;
public static class WebScraper
{
[STAThread]
public async static Task<string> LoadDynamicPage(string url, CancellationToken token)
{
using (WebBrowser webBrowser = new WebBrowser())
{
// Navigate and await DocumentCompleted
var tcs = new TaskCompletionSource<bool>();
WebBrowserDocumentCompletedEventHandler onDocumentComplete = (s, arg) => tcs.TrySetResult(true);
using (token.Register(() => tcs.TrySetCanceled(), useSynchronizationContext: true))
{
webBrowser.DocumentCompleted += onDocumentComplete;
try
{
webBrowser.Navigate(url);
await tcs.Task; // wait for DocumentCompleted
}
finally
{
webBrowser.DocumentCompleted -= onDocumentComplete;
}
}
// get the root element
var documentElement = webBrowser.Document.GetElementsByTagName("html")[0];
// poll the current HTML for changes asynchronosly
var html = documentElement.OuterHtml;
while (true)
{
// wait asynchronously, this will throw if cancellation requested
await Task.Delay(500, token);
// continue polling if the WebBrowser is still busy
if (webBrowser.IsBusy)
continue;
var htmlNow = documentElement.OuterHtml;
if (html == htmlNow)
break; // no changes detected, end the poll loop
html = htmlNow;
}
// consider the page fully rendered
token.ThrowIfCancellationRequested();
return html;
}
}
}
它目前抛出此错误
ActiveX控件'8856f961-340a-11d0-a96b-00c04fd705a2'无法实例化,因为当前线程不在单线程单元中。
我接近了吗?上面有解决方法吗?
或者如果我不在轨道上,是否有一个现成的解决方案来使用.NET获取动态Web内容(可以从类库中调用)?
这是我在Web应用程序中测试并正常工作的内容。
它在另一个线程中使用WebBrowser
控件并返回一个Task<string>
包含,当浏览器内容完全加载时完成:
using System;
using System.Threading.Tasks;
using System.Threading;
using System.Windows.Forms;
public class BrowserBasedWebScraper
{
public static Task<string> LoadUrl(string url)
{
var tcs = new TaskCompletionSource<string>();
Thread thread = new Thread(() => {
try {
Func<string> f = () => {
using (WebBrowser browser = new WebBrowser())
{
browser.ScriptErrorsSuppressed = true;
browser.Navigate(url);
while (browser.ReadyState != WebBrowserReadyState.Complete)
{
System.Windows.Forms.Application.DoEvents();
}
return browser.DocumentText;
}
};
tcs.SetResult(f());
}
catch (Exception e) {
tcs.SetException(e);
}
});
thread.SetApartmentState(ApartmentState.STA);
thread.IsBackground = true;
thread.Start();
return tcs.Task;
}
}