内涵段子、糗事百科网页抓取分析
简介：
最近写了一个小工具，用来抓取内涵段子、糗事百科等各种笑话网站的段子和图片，最后保存文本，并发布在微信公众号上。使用谷歌的 v8 做了一个脚本引擎，使用 c++ 的实现了笑话的统计和发布功能，用js实现了网页爬取分析的功能。这样 c++ 调用 v8 引擎，加载 js 脚本，就会爬取一系列的内容。
以下是网页爬取分析的内容，当然js的实现只是思路，用其他语言也是一样能实现。抓取的内容有：文章、图片地址、点赞数。
内涵段子网页抓取分析代码：
//内涵段子
//http://neihanshequ.com/
var webUrl = 'http://neihanshequ.com/';
var imageUrl = 'http://neihanshequ.com/pic/';
var index = 1;
var endIndex = 5;
var retVal =
{
success: false,
items: []};
function getJoyFromOnePage(htmlData, requestParams)
{
var nCount = 0;
var bEndOnePage = false;
while(!bEndOnePage)
{
var result =
{
webname: 'NeiHanDuanzi',
webid: '',
type: '',
context: '',
pic_url: '',
read_count: '',
publish_time: '',
best_comment: ''
}
//check gif
{
var keyWords = '"is_gif":"';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf('"');
var gif = htmlData.substring(0, endIndex);
if( gif == 1 )
{
alert('NeiHanDuanzi:url is a gif:' + result.pic_url);
continue;
}
}
}
//webid
{
var keyWords = 'data-group-id="';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf('"');
result.webid = htmlData.substring(0, endIndex);
alert('NeiHanDuanzi:webid-' + result.webid);
}
else
{
bEndOnePage = true;
alert('NeiHanDuanzi:webid not find, page end.');
}
}
//read_count
{
var keyWords = '<span class="digg">';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf('</span>');
result.read_count = htmlData.substring(0, endIndex);
//alert('NeiHanDuanzi:read_count-' + result.read_count);
}
}
//context
{
var keyWords = 'data-text="';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf('"');
result.context = htmlData.substring(0, endIndex);
//alert('NeiHanDuanzi:context-' + result.context);
}
}
//pic_url
{
var keyWords = 'data-pic="';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf('"');
result.pic_url = htmlData.substring(0, endIndex);
}
}
/*
nCount++;
if( nCount >= 20 )
{
break;
}
*/
retVal.items.push(result);
}
return;}
function getJoyContextList( url, parametersString )
{
var parameters = eval("(" + parametersString + ")");
var requestParams =
{
method: 'GET',
version: 'HTTP/1.1',
headers: {},
scriptParamaters: parameters
};
requestParams.headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8';
requestParams.headers['Accept-Encoding'] = 'gzip, deflate, sdch';
requestParams.headers['Accept-Language'] = 'zh-CN,zh;q=0.8,en;q=0.6';
requestParams.headers['Cache-Control'] = 'no-cache';
requestParams.headers['Connection'] = 'keep-alive';
requestParams.headers['Cookie'] = 'uuid="w:0ef44d961a6d43c99dd81ecb51596731"; sessionid=57f633c63c5de5d0bc03cddb0c6ee166; tt_webid=5286193655; __utmt=1; csrftoken=d760789fbe1fc31edae4ac6c11c5a700; Hm_lvt_773f1a5aa45c642cf87eef671e4d3f6a=1438825221,1438939411,1440988068,1440996740; Hm_lpvt_773f1a5aa45c642cf87eef671e4d3f6a=1440996782; __utma=101886750.2017161997.1438825217.1440995644.1440996740.6; __utmb=101886750.5.10.1440996740; __utmc=101886750; __utmz=101886750.1440996740.6.4.utmcsr=haosou.com|utmccn=(organic)|utmcmd=organic|utmctr={b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}E5{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}86{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}85{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}E6{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}B6{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}B5{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}E7{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}A4{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}BE{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}E5{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}8C{b75a474a571334bb08f4db31fa80d7688c6401b1dcf97fb55e06ed241b59472c}BA';
requestParams.headers['Host'] = 'neihanshequ.com';
requestParams.headers['Pragma'] = 'no-cache';
requestParams.headers['Referer'] = 'http://neihanshequ.com/';
requestParams.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11';
{
//get web context
var httpRspString = syncHttpRequest(webUrl, JSON.stringify(requestParams));
var httpRsp = eval("(" + httpRspString + ")");
if( !httpRsp || httpRsp.statusCode != 200 )
{
alert('NeiHanDuanzi: Request webUrl(' + webUrl + ') failed return code:' + (httpRsp ? httpRsp.statusCode : 'undefined'));
return JSON.stringify(retVal);
}
var htmlData = httpRsp.data;
getJoyFromOnePage(htmlData, requestParams)
}
{
//get image
var httpRspString = syncHttpRequest(imageUrl, JSON.stringify(requestParams));
var httpRsp = eval("(" + httpRspString + ")");
if( !httpRsp || httpRsp.statusCode != 200 )
{
alert('NeiHanDuanzi: Request imageUrl(' + imageUrl + ') failed return code:' + (httpRsp ? httpRsp.statusCode : 'undefined'));
return JSON.stringify(retVal);
}
var htmlData = httpRsp.data;
getJoyFromOnePage(htmlData, requestParams)
//alert('NeiHanDuanzi: headers' + httpRsp.data);
}
retVal.success = true;
return JSON.stringify(retVal);}
糗事百科网页抓取分析代码
//糗事百科
//http://www.qiushibaike.com/hot/page/{index}
var webUrl = 'http://www.qiushibaike.com/hot/page/';
var index = 1;
var endIndex = 5;
var retVal =
{
success: false,
items: []};
function getJoyFromOnePage(htmlData)
{
var bEndOnePage = false;
while(!bEndOnePage)
{
var result =
{
webname: 'QiuShiBaiKe',
webid: '',
type: '',
context: '',
pic_url: '',
read_count: '',
publish_time: '',
best_comment: ''
}
//webid
{
var keyWords = 'qiushi_tag_';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf("'>");
result.webid = htmlData.substring(0, endIndex);
alert('QiuShiBaiKe:webid-' + result.webid);
}
else
{
bEndOnePage = true;
alert('QiuShiBaiKe:webid not find, page end.');
}
}
//context
{
var keyWords = '<div class="content">';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf("<!");
result.context = htmlData.substring(0, endIndex);
//alert('QiuShiBaiKe:context-' + result.context);
}
}
//pic_url
{
var keyWords = '<a href="/article/' + result.webid + '" target="_blank">';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
//get sub
keyWords = '<img src="';
startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf('" alt="');
result.pic_url = htmlData.substring(0, endIndex);
//alert('QiuShiBaiKe:pic_url-' + result.pic_url);
}
}
}
//read_count
{
var keyWords = '<span class="stats-vote"><i class="number">';
var startIndex = htmlData.indexOf(keyWords);
if( startIndex > 0 )
{
htmlData = htmlData.substring((startIndex + keyWords.length));
var endIndex = htmlData.indexOf("</i>");
result.read_count = htmlData.substring(0, endIndex);
//alert('QiuShiBaiKe:read_count-' + result.read_count);
}
}
retVal.items.push(result);
}
return;}
function getJoyContextList( url, parametersString )
{
var parameters = eval("(" + parametersString + ")");
var requestParams =
{
method: 'GET',
version: 'HTTP/1.1',
headers: {},
scriptParamaters: parameters
};
requestParams.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11';
requestParams.headers['Host'] = 'www.qiushibaike.com';
requestParams.headers['Connection'] = 'keep-alive';
requestParams.headers['Accept-Encoding'] = 'gzip, deflate, sdch';
requestParams.headers['Accept-Language'] = 'zh-CN,zh;q=0.8,en;q=0.6';
requestParams.headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8';
for(var i = index; index <= endIndex; index++)
{
var trueUrl = webUrl + index;
var httpRspString = syncHttpRequest(trueUrl, JSON.stringify(requestParams));
var httpRsp = eval("(" + httpRspString + ")");
if( !httpRsp || httpRsp.statusCode != 200 )
{
alert('QiuShiBaiKe: Request trueUrl(' + trueUrl + ') failed return code:' + (httpRsp ? httpRsp.statusCode : 'undefined'));
return JSON.stringify(retVal);
}
var htmlData = httpRsp.data;
getJoyFromOnePage(htmlData)
}
retVal.success = true;
return JSON.stringify(retVal);}