Node.js+jade+mongodb+mongoose实现爬虫分离入库与生成静态文件的方法

crawlerArc(aUrl.shift());
}
}, 100);
}else {
console.log( '采集任务完成' );
return;
}
});
});
}

function filterHtml(html) {
var $ = cheerio.load(html);
var arcList = [];
var aPost = $("#content").find(".post-list-item");
aPost.each(function () {
var ele = $(this);
var title = ele.find("h2 a").text();
var url = ele.find("h2 a").attr("href");
ele.find(".c_b_p_desc a").remove();
var entry = ele.find(".c_b_p_desc").text();
ele.find("small a").remove();
var listTime = ele.find("small").text();
var re = /d{4}-d{2}-d{2}s*d{2}[:]d{2}/;
listTime = listTime.match(re)[0];

arcList.push({
title: title,
url: url,
entry: entry,
listTime: listTime
});
});
return arcList;
}

function nextPage(html) {
var $ = cheerio.load(html);
var nextUrl = $("#pager a:last-child").attr('href');
if (!nextUrl) return getArcUrl(aList);
var curPage = $("#pager .current").text();
if (!curPage) curPage = 1;
var nextPage = nextUrl.substring(nextUrl.indexOf('=') + 1);
if (curPage < nextPage) crawler(nextUrl);
}

function crawler(url) {
http.get(url, function (res) {
var html = '';
res.on('data', function (chunk) {
html += chunk;
});
res.on('end', function () {
aList.push(filterHtml(html));
nextPage(html);
});
});
}

function getArcUrl(arcList) {
for (var key in arcList) {
for (var k in arcList[key]) {
aUrl.push(arcList[key][k]['url']);
}
}
crawlerArc(aUrl.shift());
}

var url = 'http://www.cnblogs.com/ghostwu/';
crawler(url);

其他的核心模块没有怎么改动，主要增加了数据库连接，数据库创建，集合创建( 集合相当于关系型数据库中的表 )，Schema( 相当于关系型数据库的表结构 ).

mongoose操作数据库( save:插入数据 ).分离了文件生成模块.

makeHtml.js文件

var fs = require('fs');
var jade = require('jade');
var mongoose = require('mongoose');
mongoose.Promise = global.Promise;
var DB_URL = 'mongodb://localhost:27017/crawler';
var allArc = [];
var count = 0;
var db = mongoose.createConnection(DB_URL);
db.on('connected', function (err) {
 if (err) {
  console.log(err);
 } else {
  console.log('db connected success');
 }
});
var Schema = mongoose.Schema;
var arcSchema = new Schema({
 id: Number, //文章id
 title: String, //文章标题
 url: String, //文章链接
 body: String, //文章内容
 entry: String, //摘要
 listTime: Date //发布时间
});
var Article = db.model('Article', arcSchema);
function makeHtml(arcDetail) {
 str = jade.renderFile('./views/layout.jade', arcDetail);								 
 2/3   首页 上一页 1 2 3 下一页 尾页