c#批量抓取免费代理并且验证有效性的实战教程

3.拿到代理集合，多线程发起http请求，比如访问百度，是否成功，成功则存到Redis里面。

效果图如下：

使用HttpWebRequest发起请求

Request.cs如下，主要就是两个方法，一个方法是验证代理ip是否有效，设置HttpWebRequest的Proxy属性，请求百度，看到有些文章大多数会获取响应的内容，如果内容符合请求的网址则证明代理哟有效，实际上根据HttpStatusCode 200就可以判断是否验证有效。

【注意】建的是控制台程序，使用了异步，所以还是建.net core吧，c#语言的版本7.1。

public class Request
 {
 /// <summary>
 /// 验证代理ip有效性
 /// </summary>
 /// <param name="proxyIp">代理IP</param>
 /// <param name="proxyPort">代理IP 端口</param>
 /// <param name="timeout">详情超时</param>
 /// <param name="url">请求的地址</param>
 /// <param name="success">成功的回调</param>
 /// <param name="fail">失败的回调</param>
 /// <returns></returns>
 public static async System.Threading.Tasks.Task getAsync(string proxyIp,int proxyPort, int timeout,string url, Action success, Action<string> fail)
 {
  System.GC.Collect();
  HttpWebRequest request = null;
  HttpWebResponse response = null;
  try
  {
  request = (HttpWebRequest)WebRequest.Create(url);
  //HttpWebRequest request = HttpWebRequest.CreateHttp(url);
  request.Timeout =timeout;
  request.KeepAlive = false;
  request.Proxy = new WebProxy(proxyIp,proxyPort);
  response = await request.GetResponseAsync() as HttpWebResponse;
  if (response.StatusCode == HttpStatusCode.OK)
  {
   success();
  }
  else
  {
   fail(response.StatusCode+":"+response.StatusDescription);
  }
  }
  catch (Exception ex)
  {
  fail("请求异常"+ex.Message.ToString());
  }
  finally
  {
  if (request != null)
  {
   request.Abort();
   request = null;
  }
  if (response != null)
  {
   response.Close();
  }
  }
 }

 /// <summary>
 /// 发起http请求
 /// </summary>
 /// <param name="url"></param>
 /// <param name="success">成功的回调</param>
 /// <param name="fail">失败的回调</param>
 public static void get(string url,Action<string> success,Action<string> fail)
 {
  StreamReader reader = null;
  Stream stream = null;
  WebRequest request = null;
  HttpWebResponse response = null;
  try
  {
  request = WebRequest.Create(url);
  request.Timeout = 2000;
  response = (HttpWebResponse)request.GetResponse();
  if (response.StatusCode == HttpStatusCode.OK)
  {
   stream = response.GetResponseStream();
   reader = new StreamReader(stream);
   string result = reader.ReadToEnd();
   success(result);
  }
  else
  {
   fail(response.StatusCode+":"+response.StatusDescription);
  }
  }
  catch (Exception ex)
  {
  fail(ex.ToString());
  }
  finally
  {
  if (reader != null)
   reader.Close();
  if (stream != null)
   stream.Close();
  if(response!=null)
   response.Close();
  if(request!=null)
   request.Abort();
  }
 }
 }								 
 3/6   首页 上一页 1 2 3 4 5 6 下一页 尾页