c#爬虫爬取京东的商品信息

获取所有商品的信息分为两个步骤

    （1）根据商品分类页面获取所有商品分类的URL

    （2）根据商品分类URL获取每个商品

 1、获取商品分类

try
{
 string html = HttpHelper.DownloadUrl(@"http://www.easck.com/allSort.aspx");
 HtmlDocument doc = new HtmlDocument();
 doc.LoadHtml(html);
 string goodClass= @"//*[@class='items']/dl/dd";
 HtmlNodeCollection noneNodeList = doc.DocumentNode.SelectNodes(goodClass);
 foreach (var node in noneNodeList)
 {
 HtmlDocument docChild = new HtmlDocument();
 docChild.LoadHtml(node.OuterHtml);
 string urlPath = "/dd/a";
 HtmlNodeCollection list = docChild.DocumentNode.SelectNodes(urlPath);
 foreach (var l in list)
 {
  HtmlDocument docChild1 = new HtmlDocument();
  docChild1.LoadHtml(l.OuterHtml);
  var sortUrl = l.Attributes["href"].Value;
  if (!string.IsNullOrWhiteSpace(sortUrl) && sortUrl.Contains("cat="))
  {
  InsertSort("https:" + sortUrl);
  }
 }
 }
}
catch (Exception ex)
{
 Console.WriteLine(ex.Message);
}