Java Jsoup Spider抓取数据入库

package tk.mybatis.springboot.util;import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.mysql.jdbc.Connection;
import com.mysql.jdbc.PreparedStatement;
import tk.mybatis.springboot.model.AutobatchDirectory;
import tk.mybatis.springboot.service.AutobatchDirectoryService;public class AutoBatchSpider { // 原始来源http://www.cn357.com/notice_list/
public static final String web = "http://www.cn357.com"; private static final int timeOut=30000; /**
* 获取汽车公告批次
*
* @throws InterruptedException
* @throws IOException
*
*/
public static void getBatchFromUrl(String listurl) throws InterruptedException, IOException {
Document doc = Jsoup.connect(listurl).userAgent("Mozilla/5.0").timeout(timeOut).get();
Element batchElement = doc.getElementById("noticeList");
Elements elements = batchElement.children();
List<String> urls = new ArrayList<String>();
for (Element element : elements) {
String href = element.attr("href");
String text = element.text();
if (!href.startsWith("http://")) {
StringBuffer sb = new StringBuffer();
String batchUrl = sb.append(web).append(href).toString();
System.out.println(text + "\t\t" + batchUrl);
urls.add(batchUrl);
} else {
String batchUrl = href;
urls.add(batchUrl);
System.out.println(text + "\t\t" + batchUrl);
}
}
// 反转排序
Collections.reverse(urls); System.out.println("总批次数:" + urls.size());
for (int j = 0, k = urls.size(); j < k; j++) {
String url = urls.get(j);
System.out.println("第" + url.split("_")[1] + "批数据获取进度：" + (double) Math.round(j * 100 / k) / 100);
// 获取分页链接
List<Map<String,Object>> list=getDetailsPageFromBatchItems(url);
for (Map<String, Object> map : list) {
// 获取详细数据对象
String detailUrl =(String)map.get("href");
AutobatchDirectory autobatchDirectory=getDetailOfAutoBatchInfo(detailUrl);
try {
saveByJdbc(autobatchDirectory);
} catch (SQLException e) {
e.printStackTrace();
}
}
} } /**
* 获取汽车公告批次
*
* @throws InterruptedException
* @throws IOException
*
*/
public static void getBatchFromUrl(AutobatchDirectoryService autobatchDirectoryService,String listurl) throws InterruptedException, IOException {
Document doc = Jsoup.connect(listurl).userAgent("Mozilla/5.0").timeout(timeOut).get();
Element batchElement = doc.getElementById("noticeList");
Elements elements = batchElement.children();
List<String> urls = new ArrayList<String>();
for (Element element : elements) {
String href = element.attr("href");
String text = element.text();
if (!href.startsWith("http://")) {
StringBuffer sb = new StringBuffer();
String batchUrl = sb.append(web).append(href).toString();
System.out.println(text + "\t\t" + batchUrl);
urls.add(batchUrl);
} else {
String batchUrl = href;
urls.add(batchUrl);
System.out.println(text + "\t\t" + batchUrl);
}
}
// 反转排序
Collections.reverse(urls); System.out.println("总批次数:" + urls.size());
for (int j = 0, k = urls.size(); j < k; j++) {
String url = urls.get(j);
System.out.println("第" + url.split("_")[1] + "批数据获取进度：" + (double) Math.round(j * 100 / k) / 100);
// 获取分页链接
List<Map<String,Object>> list=getDetailsPageFromBatchItems(url);
for (Map<String, Object> map : list) {
// 获取详细数据对象
String detailUrl =(String)map.get("href");
AutobatchDirectory autobatchDirectory=getDetailOfAutoBatchInfo(detailUrl);
autobatchDirectoryService.save(autobatchDirectory);
}
} } /**
* 获取所有汽车公告批次详细分页条目
*
* @throws InterruptedException
* @throws IOException
*
*/
public static List<Map<String, Object>> getDetailsPageFromBatchItems(String url)
throws InterruptedException, IOException { Document doc = Jsoup.connect(url).userAgent("Mozilla/5.0").timeout(timeOut).get();
Thread.sleep(1000);
/************ 获取分页参数 ************/
Elements pages = doc.getElementsByAttributeValue("class", "page");
int max = 1;
for (Element element : pages) {
for (Element children : element.getElementsByTag("a")) {
String text = children.text();
if (!text.equals("") && !text.equals("下一页") && !text.equals("上一页")) {
int index = Integer.valueOf(children.text());
if (index > max) {
max = index;
}
}
}
}
int totalBatchPage = max;
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
/************ 获取分页数据 ************/
for (int i = 1; i <= totalBatchPage; i++) {
System.out.println("分页数据获取进度：" + (double) Math.round(i * 100 / max) / 100);
Thread.sleep(2000);// 等待2秒开始访问
doc = Jsoup.connect(url + "_" + i).userAgent("Mozilla/5.0").timeout(timeOut).get();
Elements divs = doc.getElementsByAttributeValue("class", "noticeLotItem");
Map<String, Object> map = new HashMap<String, Object>();
for (Element div : divs) {
Elements divChildren = div.children();
boolean isHref = false;
for (Element element : divChildren) {
String claszType = element.attr("class");
if (claszType.equals("m")) {
Element a = element.getElementsByAttribute("href").get(0);
String href = a.attr("href");
if ("".equals(href)) {
continue;
}
map = new HashMap<String, Object>();
if (!href.startsWith("http://")) {
map.put("href", web + href);
} else {
map.put("href", href);
}
map.put("href_text", a.text());
isHref = true;
} else if (claszType.equals("c")) {
if (!isHref) {
continue;
}
map.put("type_text", element.text());
}
}
if (map != null) {
list.add(map);
map = null;
}
}
}
System.out.println("Total rows:" + list.size());
return list;
} /**
* 解析详细的车型车厂信息
*
* @throws IOException
* @throws InterruptedException
*/
public static AutobatchDirectory getDetailOfAutoBatchInfo(String url) throws IOException, InterruptedException {
Thread.sleep(2000);
Document doc = Jsoup.connect(url).userAgent("Mozilla/5.0").timeout(timeOut).get();
Elements tableElements = doc.getElementsByAttributeValue("class", "noticeAttr mt5").get(0)
.getElementsByTag("tbody").get(0).children();
AutobatchDirectory autobatchDirectory = null;
if (tableElements.size() == 22) {
autobatchDirectory = new AutobatchDirectory();
// 22行表示没有发动机参数
System.out.println("......22行表示没有发动机参数.....解析中.......");
for (int i = 1, j = tableElements.size(); i <= j; i++) {
Element element = tableElements.get(i - 1);
if (i == j) {
Elements children = element.children();
String values = children.get(1).text();
autobatchDirectory.setfRemark(values);
System.out.println(values);
} else {
// 设置对象属性值
setPropertyToObject(autobatchDirectory, i, element);
}
}
} else if (tableElements.size() == 23) {
autobatchDirectory = new AutobatchDirectory();
// 23行表示有发动机参数
System.out.println("......23行表示有发动机参数.....解析中.......");
for (int i = 1, j = tableElements.size(); i <= j; i++) {
Element element = tableElements.get(i - 1);
if (i == 22) {
Element valueTr = element.getElementsByTag("table").get(0);
Elements children = valueTr.getElementsByTag("tbody").get(0).children().get(1).children();
int count = 0;
for (Element child : children) {
switch (count) {
case 0:
// 发动机型号
autobatchDirectory.setfEngineType(child.text());
break;
case 1:
// 发动机生产企业
autobatchDirectory.setfEnginePro(child.text());
break;
case 2:
// 发动机商标
autobatchDirectory.setfEngineTrademark(child.text());
break;
case 3:
// 排量
autobatchDirectory.setfOutputVolume(child.text());
break;
case 4:
// 功率
autobatchDirectory.setfPower(child.text());
break;
default:
break;
}
count++;
}
} else if (i == j) {
Elements children = element.children();
String values = children.get(1).text();
autobatchDirectory.setfRemark(values);
System.out.println(values);
} else {
// 设置对象属性值
setPropertyToObject(autobatchDirectory, i, element);
}
}
}
return autobatchDirectory;
} /**
* 设置对象属性值
*/
private static void setPropertyToObject(AutobatchDirectory autobatchDirectory, int i, Element element) {
Elements children = element.children();
String values = children.get(1).text() + " " + children.get(3).text();
System.out.println(values);
switch (i) {
case 1:
// 公告型号 公告批次
autobatchDirectory.setfAnnouType(children.get(1).text());
autobatchDirectory.setfAnnouBatch(children.get(3).text());
break;
case 2:
// 品牌 类型
autobatchDirectory.setfVehicleBrand(children.get(1).text());
autobatchDirectory.setfVehicleType(children.get(3).text());
break;
case 3:
// 额定质量 32000,32700 总质量
autobatchDirectory.setfMaxMass(children.get(1).text());
autobatchDirectory.setfTotalMass(children.get(3).text());
break;
case 4:
// 整备质量 8000,7300 燃料种类
autobatchDirectory.setfWholeMass(children.get(1).text());
autobatchDirectory.setfFuelType(children.get(3).text());
break;
case 5:
// 排放依据标准 轴数
autobatchDirectory.setfBlowoffStandard(children.get(1).text());
autobatchDirectory.setfAxleNumber(children.get(3).text());
break;
case 6:
// 轴距 7250+1310+1310,6850+1310+1310 轴荷
autobatchDirectory.setfWheelbase(children.get(1).text());
autobatchDirectory.setfAxleWeight(children.get(3).text());
break;
case 7:
// 弹簧片数 -/8/8/8,-/4/4/4,-/7/7/7,-/-/-/-,-/10/10/10 轮胎数
autobatchDirectory.setfSpringNumber(children.get(1).text());
autobatchDirectory.setfTyreNumber(children.get(3).text());
break;
case 8:
// 轮胎规格 11.00R20 12PR,11.00-20 12PR,12R22.5 12PR 接近离去角
autobatchDirectory.setfTyreSize(children.get(1).text());
autobatchDirectory.setfDepartureAngle(children.get(3).text());
break;
case 9:
// 前悬后悬 -/2080,-/1730,-/2480,-/2130 前轮距
autobatchDirectory.setfFrearSuspension(children.get(1).text());
autobatchDirectory.setfFrontGauge(children.get(3).text());
break;
case 10:
// 后轮距 1830/1830/1830 识别代号
autobatchDirectory.setfBackGauge(children.get(1).text());
autobatchDirectory.setfVinCode(children.get(3).text());
break;
case 11:
// 整车长 13000 整车宽
autobatchDirectory.setfVehicleLength(children.get(1).text());
autobatchDirectory.setfVehicleWidth(children.get(3).text());
break;
case 12:
// 整车高 2970,3030,2760 货厢长
autobatchDirectory.setfVehicleHeight(children.get(1).text());
autobatchDirectory.setfCargoLength(children.get(3).text());
break;
case 13:
// 货厢宽 2400,2450,2470 货厢高
autobatchDirectory.setfCargoWidth(children.get(1).text());
autobatchDirectory.setfCargoHeight(children.get(3).text());
break;
case 14:
// 最高车速 额定载客
autobatchDirectory.setfMaxSpeed(children.get(1).text());
autobatchDirectory.setfMaxPassenger(children.get(3).text());
break;
case 15:
// 驾驶室准乘人数 转向形式
autobatchDirectory.setfCabNumber(children.get(1).text());
autobatchDirectory.setfSteeringType(children.get(3).text());
break;
case 16:
// 准拖挂车总质量 载质量利用系数
autobatchDirectory.setfTotalMassTrailer(children.get(1).text());
autobatchDirectory.setfLoadMassFactor(children.get(3).text());
break;
case 17:
// 半挂车鞍座最大承载质量 16000,16150 企业名称
autobatchDirectory.setfMaxSemitrailer(children.get(1).text());
autobatchDirectory.setfEnterpriseName(children.get(3).text());
break;
case 18:
// 企业地址 深圳市龙岗区坪山镇锦龙大道1号 电话号码
autobatchDirectory.setfEnterpriseAddress(children.get(1).text());
autobatchDirectory.setfEnterprisePhone(children.get(3).text());
break;
case 19:
// 传真号码 (0755)89663298 邮政编码
autobatchDirectory.setfEnterpriseFax(children.get(1).text());
autobatchDirectory.setfPostcode(children.get(3).text());
break;
case 20:
// 底盘1 底盘2
autobatchDirectory.setfChassisOne(children.get(1).text());
autobatchDirectory.setfChassisTwo(children.get(3).text());
break;
case 21:
// 底盘3 底盘4
autobatchDirectory.setfChassisThree(children.get(1).text());
autobatchDirectory.setfChassisFour(children.get(3).text());
break;
default:
break;
}
} /**
* JDBC存储
* @throws SQLException
*/
public static void saveByJdbc(AutobatchDirectory autobatchDirectory) throws SQLException{
String sql = "insert into autobatch_directory ("
+ "F_ANNOU_TYPE, F_ANNOU_BATCH, F_VEHICLE_BRAND, F_VEHICLE_TYPE,"
+ "F_MAX_MASS, F_TOTAL_MASS, F_WHOLE_MASS, F_FUEL_TYPE, "
+ "F_BLOWOFF_STANDARD, F_AXLE_NUMBER, F_WHEELBASE, F_AXLE_WEIGHT,"
+ "F_SPRING_NUMBER, F_TYRE_NUMBER,F_TYRE_SIZE, F_DEPARTURE_ANGLE, "
+ "F_FREAR_SUSPENSION, F_FRONT_GAUGE, F_BACK_GAUGE, F_VIN_CODE, "
+ "F_VEHICLE_LENGTH, F_VEHICLE_WIDTH, F_VEHICLE_HEIGHT,F_CARGO_LENGTH,"
+ "F_CARGO_WIDTH, F_CARGO_HEIGHT,F_MAX_SPEED, F_MAX_PASSENGER,"
+ "F_CAB_NUMBER, F_STEERING_TYPE, F_TOTAL_MASS_TRAILER,"
+ "F_LOAD_MASS_FACTOR, F_MAX_SEMITRAILER, F_ENTERPRISE_NAME, F_ENTERPRISE_ADDRESS, "
+ "F_ENTERPRISE_PHONE, F_ENTERPRISE_FAX, F_POSTCODE, F_CHASSIS_ONE,"
+ "F_CHASSIS_TWO, F_CHASSIS_THREE, F_CHASSIS_FOUR, F_ENGINE_TYPE,"
+ "F_ENGINE_PRO, F_ENGINE_TRADEMARK, F_OUTPUT_VOLUME, F_POWER, F_REMARK) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,"
+ "?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
Connection conn = MycatJdbc.getConnection();
PreparedStatement pstm = null;
try {
System.out.println(sql);
pstm = (PreparedStatement) conn.prepareStatement(sql);
pstm.setString(1, autobatchDirectory.getfAnnouType());
pstm.setString(2, autobatchDirectory.getfAnnouBatch());
pstm.setString(3, autobatchDirectory.getfVehicleBrand());
pstm.setString(4, autobatchDirectory.getfVehicleType());
pstm.setString(5, autobatchDirectory.getfMaxMass());
pstm.setString(6, autobatchDirectory.getfTotalMass());
pstm.setString(7, autobatchDirectory.getfWholeMass());
pstm.setString(8, autobatchDirectory.getfFuelType());
pstm.setString(9, autobatchDirectory.getfBlowoffStandard());
pstm.setString(10, autobatchDirectory.getfAxleNumber());
pstm.setString(11, autobatchDirectory.getfWheelbase());
pstm.setString(12,autobatchDirectory.getfAxleWeight() );
pstm.setString(13, autobatchDirectory.getfSpringNumber());
pstm.setString(14,autobatchDirectory.getfTyreNumber() );
pstm.setString(15, autobatchDirectory.getfTyreSize() );
pstm.setString(16, autobatchDirectory.getfDepartureAngle());
pstm.setString(17, autobatchDirectory.getfFrearSuspension());
pstm.setString(18, autobatchDirectory.getfFrontGauge());
pstm.setString(19,autobatchDirectory.getfBackGauge() );
pstm.setString(20, autobatchDirectory.getfVinCode());
pstm.setString(21, autobatchDirectory.getfVehicleLength());
pstm.setString(22, autobatchDirectory.getfVehicleWidth());
pstm.setString(23, autobatchDirectory.getfVehicleHeight());
pstm.setString(24, autobatchDirectory.getfCargoLength());
pstm.setString(25,autobatchDirectory.getfCargoWidth() );
pstm.setString(26, autobatchDirectory.getfCargoHeight());
pstm.setString(27,autobatchDirectory.getfMaxSpeed() );
pstm.setString(28,autobatchDirectory.getfMaxPassenger() );
pstm.setString(29,autobatchDirectory.getfCabNumber() );
pstm.setString(30, autobatchDirectory.getfSteeringType() );
pstm.setString(31, autobatchDirectory.getfTotalMassTrailer());
pstm.setString(32,autobatchDirectory.getfLoadMassFactor() );
pstm.setString(33,autobatchDirectory.getfMaxSemitrailer() );
pstm.setString(34, autobatchDirectory.getfEnterpriseName());
pstm.setString(35,autobatchDirectory.getfEnterpriseAddress() );
pstm.setString(36,autobatchDirectory.getfEnterprisePhone() );
pstm.setString(37,autobatchDirectory.getfEnterpriseFax() );
pstm.setString(38, autobatchDirectory.getfPostcode() );
pstm.setString(39, autobatchDirectory.getfChassisOne());
pstm.setString(40, autobatchDirectory.getfChassisTwo());
pstm.setString(41, autobatchDirectory.getfChassisThree());
pstm.setString(42, autobatchDirectory.getfChassisFour());
pstm.setString(43, autobatchDirectory.getfEngineType() == null ? "" : autobatchDirectory.getfEngineType());
pstm.setString(44, autobatchDirectory.getfEnginePro() == null ? "" : autobatchDirectory.getfEnginePro());
pstm.setString(45, autobatchDirectory.getfEngineTrademark() == null ? "" : autobatchDirectory.getfEngineTrademark());
pstm.setString(46, autobatchDirectory.getfOutputVolume() == null ? "" : autobatchDirectory.getfOutputVolume());
pstm.setString(47, autobatchDirectory.getfPower() == null ? "" : autobatchDirectory.getfPower());
pstm.setString(48, autobatchDirectory.getfRemark()==null?"":autobatchDirectory.getfRemark());
pstm.executeUpdate();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (pstm != null) {
try {
pstm.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
}