Staging to Production #51
@@ -117,6 +117,10 @@ public sealed class HtmlJobSearcher
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip non-HTTP(S) URLs (e.g. file:// or javascript: that can appear in scraped HTML)
|
||||
if (absoluteUri.Scheme != Uri.UriSchemeHttp && absoluteUri.Scheme != Uri.UriSchemeHttps)
|
||||
continue;
|
||||
|
||||
var url = absoluteUri.GetLeftPart(UriPartial.Path);
|
||||
if (seen.Add(url))
|
||||
results.Add(url);
|
||||
|
||||
Reference in New Issue
Block a user