Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2e9069cbdb |
@@ -117,6 +117,10 @@ public sealed class HtmlJobSearcher
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip non-HTTP(S) URLs (e.g. file:// or javascript: that can appear in scraped HTML)
|
||||||
|
if (absoluteUri.Scheme != Uri.UriSchemeHttp && absoluteUri.Scheme != Uri.UriSchemeHttps)
|
||||||
|
continue;
|
||||||
|
|
||||||
var url = absoluteUri.GetLeftPart(UriPartial.Path);
|
var url = absoluteUri.GetLeftPart(UriPartial.Path);
|
||||||
if (seen.Add(url))
|
if (seen.Add(url))
|
||||||
results.Add(url);
|
results.Add(url);
|
||||||
|
|||||||
Reference in New Issue
Block a user