java的爬虫代码怎么写 java的爬虫代码怎么写出来

用java编写 网络爬虫求代码和流程 急

import java.awt.*;

import java.awt.event.*;

import java.io.*;

import java.net.*;

import java.util.*;

import java.util.regex.*;

import javax.swing.*;

import javax.swing.table.*;//一个Web的爬行者(注：爬行在这里的意思与抓取，捕获相同)

public class SearchCrawler extends JFrame{

//最大URL保存值

private static final String[] MAX_URLS={"50","100","500","1000"};

//缓存robot禁止爬行列表

private HashMap disallowListCache=new HashMap();

//搜索GUI控件

private JTextField startTextField;

private JComboBox maxComboBox;

private JCheckBox limitCheckBox;

private JTextField logTextField;

private JTextField searchTextField;

private JCheckBox caseCheckBox;

private JButton searchButton;

//搜索状态GUI控件

private JLabel crawlingLabel2;

private JLabel crawledLabel2;

private JLabel toCrawlLabel2;

private JProgressBar progressBar;

private JLabel matchesLabel2;

//搜索匹配项表格列表

private JTable table;

//标记爬行机器是否正在爬行

private boolean crawling;

//写日志匹配文件的引用

private PrintWriter logFileWriter;

//网络爬行者的构造函数

public SearchCrawler(){

//设置应用程序标题栏

setTitle("搜索爬行者");

//设置窗体大小

setSize(600,600);

//处理窗体关闭事件

addWindowListener(new WindowAdapter(){

public void windowClosing(WindowEvent e){

actionExit();

}

});

//设置文件菜单

JMenuBar menuBar=new JMenuBar();

JMenu fileMenu=new JMenu("文件");

fileMenu.setMnemonic(KeyEvent.VK_F);

JMenuItem fileExitMenuItem=new JMenuItem("退出",KeyEvent.VK_X);

fileExitMenuItem.addActionListener(new ActionListener(){

public void actionPerformed(ActionEvent e){

actionExit();

}

});

fileMenu.add(fileExitMenuItem);

menuBar.add(fileMenu);

setJMenuBar(menuBar);

相关问答

Q1: Java网络爬虫怎么实现？

网络爬虫是一个自动提取网页java的爬虫代码怎么写的程序java的爬虫代码怎么写，它为搜索引擎从万维网上下载网页java的爬虫代码怎么写，是搜索引擎的重要组成。\x0d\x0a传统爬虫从一个或若干初始网页的URL开始，获得初始网页上的URL，在抓取网页的过程中，不断从当前页面上抽取新的URL放入队列，直到满足系统的一定停止条件。对于垂直搜索来说，聚焦爬虫，即有针对性地爬取特定主题网页的爬虫，更为适合。\x0d\x0a\x0d\x0a以下是一个使用java实现的简单爬虫核心代码java的爬虫代码怎么写：\x0d\x0apublic void crawl() throws Throwable { \x0d\x0a while (continueCrawling()) { \x0d\x0a CrawlerUrl url = getNextUrl(); //获取待爬取队列中的下一个URL \x0d\x0a if (url != null) { \x0d\x0a printCrawlInfo(); \x0d\x0a String content = getContent(url); //获取URL的文本信息 \x0d\x0a \x0d\x0a //聚焦爬虫只爬取与主题内容相关的网页，这里采用正则匹配简单处理 \x0d\x0a if (isContentRelevant(content, this.regexpSearchPattern)) { \x0d\x0a saveContent(url, content); //保存网页至本地 \x0d\x0a \x0d\x0a //获取网页内容中的链接，并放入待爬取队列中 \x0d\x0a Collection urlStrings = extractUrls(content, url); \x0d\x0a addUrlsToUrlQueue(url, urlStrings); \x0d\x0a } else { \x0d\x0a System.out.println(url + " is not relevant ignoring ..."); \x0d\x0a } \x0d\x0a \x0d\x0a //延时防止被对方屏蔽 \x0d\x0a Thread.sleep(this.delayBetweenUrls); \x0d\x0a } \x0d\x0a } \x0d\x0a closeOutputStream(); \x0d\x0a}\x0d\x0aprivate CrawlerUrl getNextUrl() throws Throwable { \x0d\x0a CrawlerUrl nextUrl = null; \x0d\x0a while ((nextUrl == null) (!urlQueue.isEmpty())) { \x0d\x0a CrawlerUrl crawlerUrl = this.urlQueue.remove(); \x0d\x0a //doWeHavePermissionToVisitjava的爬虫代码怎么写：是否有权限访问该URL，友好的爬虫会根据网站提供的"Robot.txt"中配置的规则进行爬取 \x0d\x0a //isUrlAlreadyVisited：URL是否访问过，大型的搜索引擎往往采用BloomFilter进行排重，这里简单使用HashMap \x0d\x0a //isDepthAcceptable：是否达到指定的深度上限。爬虫一般采取广度优先的方式。一些网站会构建爬虫陷阱（自动生成一些无效链接使爬虫陷入死循环），采用深度限制加以避免 \x0d\x0a if (doWeHavePermissionToVisit(crawlerUrl) \x0d\x0a (!isUrlAlreadyVisited(crawlerUrl)) \x0d\x0a isDepthAcceptable(crawlerUrl)) { \x0d\x0a nextUrl = crawlerUrl; \x0d\x0a // System.out.println("Next url to be visited is " + nextUrl); \x0d\x0a } \x0d\x0a } \x0d\x0a return nextUrl; \x0d\x0a}\x0d\x0aprivate String getContent(CrawlerUrl url) throws Throwable { \x0d\x0a //HttpClient4.1的调用与之前的方式不同 \x0d\x0a HttpClient client = new DefaultHttpClient(); \x0d\x0a HttpGet httpGet = new HttpGet(url.getUrlString()); \x0d\x0a StringBuffer strBuf = new StringBuffer(); \x0d\x0a HttpResponse response = client.execute(httpGet); \x0d\x0a if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) { \x0d\x0a HttpEntity entity = response.getEntity(); \x0d\x0a if (entity != null) { \x0d\x0a BufferedReader reader = new BufferedReader( \x0d\x0a new InputStreamReader(entity.getContent(), "UTF-8")); \x0d\x0a String line = null; \x0d\x0a if (entity.getContentLength() 0) { \x0d\x0a strBuf = new StringBuffer((int) entity.getContentLength()); \x0d\x0a while ((line = reader.readLine()) != null) { \x0d\x0a strBuf.append(line); \x0d\x0a } \x0d\x0a } \x0d\x0a } \x0d\x0a if (entity != null) { \x0d\x0a nsumeContent(); \x0d\x0a } \x0d\x0a } \x0d\x0a //将url标记为已访问 \x0d\x0a markUrlAsVisited(url); \x0d\x0a return strBuf.toString(); \x0d\x0a}\x0d\x0apublic static boolean isContentRelevant(String content, \x0d\x0aPattern regexpPattern) { \x0d\x0a boolean retValue = false; \x0d\x0a if (content != null) { \x0d\x0a //是否符合正则表达式的条件 \x0d\x0a Matcher m = regexpPattern.matcher(content.toLowerCase()); \x0d\x0a retValue = m.find(); \x0d\x0a } \x0d\x0a return retValue; \x0d\x0a}\x0d\x0apublic List extractUrls(String text, CrawlerUrl crawlerUrl) { \x0d\x0a Map urlMap = new HashMap(); \x0d\x0a extractHttpUrls(urlMap, text); \x0d\x0a extractRelativeUrls(urlMap, text, crawlerUrl); \x0d\x0a return new ArrayList(urlMap.keySet()); \x0d\x0a} \x0d\x0aprivate void extractHttpUrls(Map urlMap, String text) { \x0d\x0a Matcher m = (text); \x0d\x0a while (m.find()) { \x0d\x0a String url = m.group(); \x0d\x0a String[] terms = url.split("a href=\""); \x0d\x0a for (String term : terms) { \x0d\x0a // System.out.println("Term = " + term); \x0d\x0a if (term.startsWith("http")) { \x0d\x0a int index = term.indexOf("\""); \x0d\x0a if (index 0) { \x0d\x0a term = term.substring(0, index); \x0d\x0a } \x0d\x0a urlMap.put(term, term); \x0d\x0a System.out.println("Hyperlink: " + term); \x0d\x0a } \x0d\x0a } \x0d\x0a } \x0d\x0a} \x0d\x0aprivate void extractRelativeUrls(Map urlMap, String text, \x0d\x0a CrawlerUrl crawlerUrl) { \x0d\x0a Matcher m = relativeRegexp.matcher(text); \x0d\x0a URL textURL = crawlerUrl.getURL(); \x0d\x0a String host = textURL.getHost(); \x0d\x0a while (m.find()) { \x0d\x0a String url = m.group(); \x0d\x0a String[] terms = url.split("a href=\""); \x0d\x0a for (String term : terms) { \x0d\x0a if (term.startsWith("/")) { \x0d\x0a int index = term.indexOf("\""); \x0d\x0a if (index 0) { \x0d\x0a term = term.substring(0, index); \x0d\x0a } \x0d\x0a String s = //" + host + term; \x0d\x0a urlMap.put(s, s); \x0d\x0a System.out.println("Relative url: " + s); \x0d\x0a } \x0d\x0a } \x0d\x0a } \x0d\x0a \x0d\x0a}\x0d\x0apublic static void main(String[] args) { \x0d\x0a try { \x0d\x0a String url = ""; \x0d\x0a Queue urlQueue = new LinkedList(); \x0d\x0a String regexp = "java"; \x0d\x0a urlQueue.add(new CrawlerUrl(url, 0)); \x0d\x0a NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L, \x0d\x0a regexp); \x0d\x0a // boolean allowCrawl = crawler.areWeAllowedToVisit(url); \x0d\x0a // System.out.println("Allowed to crawl: " + url + " " + \x0d\x0a // allowCrawl); \x0d\x0a crawler.crawl(); \x0d\x0a } catch (Throwable t) { \x0d\x0a System.out.println(t.toString()); \x0d\x0a t.printStackTrace(); \x0d\x0a } \x0d\x0a}

Q2: 请教用java编写一个网络爬虫程序

你说说你都要什么功能啊，要是功能简单我就给你做，复杂你得给点钱

Q3: 如何用Java写一个爬虫

最近刚好在学这个java的爬虫代码怎么写，对于一些第三方工具类或者库java的爬虫代码怎么写，一定要看官方tutorial啊。

学会用chrome network 分析请求，或者fiddler抓包分析。

普通java的爬虫代码怎么写的网页直接用httpclient封装的API就可以获取网页HTMLjava的爬虫代码怎么写了，然后 JSoup、正则 提取内容。

若网站有反爬虫机制的，会需要构造User-Agent 伪装浏览器； 若有需要登录的，会传入cookie进去。

有些网页可能是利用ajax技术的，可以尝试PhantomJS拿到渲染后的HTML(然后步骤同上); 或者直接chrome network分析请求的URL以及传参，然后直接拿到json。

关于图片验证码的没尝试过，是不是要用到第三方ocr工具识别图片验证码然后作为URL参数，希望做过的童鞋告知。

Q4: 用java编写网络爬虫，用来爬网络音乐资源，再返回java页面显示该怎么实现

下面是源代码,希望可以帮到你~~

package com.ly.mainprocess;

import java.io.BufferedReader;

import java.io.InputStreamReader;

import java.util.ArrayList;

import java.util.List;

import org.apache.http.Consts;

import org.apache.http.Header;

import org.apache.http.HttpEntity;

import org.apache.http.HttpResponse;

import org.apache.http.NameValuePair;

import org.apache.http.StatusLine;

import org.apache.http.client.entity.UrlEncodedFormEntity;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.cookie.Cookie;

import org.apache.http.impl.client.DefaultHttpClient;

import org.apache.http.message.BasicNameValuePair;

import org.apache.http.util.EntityUtils;

public class Test1 {

public static void main(String[] args){

Test1 test1 = new Test1();

System.out.println(test1.process("******","******"));

}

@SuppressWarnings("deprecation")

public boolean process(String username,String password) {

boolean ret=false;

DefaultHttpClient httpclient = new DefaultHttpClient();

try {

HttpGet httpget;

HttpResponse response;

HttpEntity entity;

ListCookie cookies;

//组建登录的post包

HttpPost httppost = new HttpPost(""); // 用户登录

ListNameValuePair nvps = new ArrayListNameValuePair();

nvps.add(new BasicNameValuePair("nickname", username));

nvps.add(new BasicNameValuePair("password", password));

nvps.add(new BasicNameValuePair("origURL", ""));

nvps.add(new BasicNameValuePair("loginregFrom", "index"));

nvps.add(new BasicNameValuePair("ss", "10101"));

httppost.setEntity(new UrlEncodedFormEntity(nvps, Consts.UTF_8));

httppost.addHeader("Referer", "");

httppost.addHeader("Connection", "keep-alive");

httppost.addHeader("Content-Type", "application/x-www-form-urlencoded");

httppost.addHeader("Accept-Language", "zh-CN,zh;q=0.8");

httppost.addHeader("Origin", "");

httppost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");

response = httpclient.execute(httppost);

entity = response.getEntity();

// System.out.println("Login form get: " + response.getStatusLine());

EntityUtils.consume(entity);

// System.out.println("Post logon cookies:");

cookies = httpclient.getCookieStore().getCookies();

if (cookies.isEmpty()) {

// System.out.println("None");

} else {

for (int i = 0; i cookies.size(); i++) {

// System.out.println("- " + cookies.get(i).toString());

}

}

//进行页面跳转

String url = ""; // 页面跳转

Header locationHeader = response.getFirstHeader("Location");

// System.out.println(locationHeader.getValue());

if (locationHeader != null) {

url = locationHeader.getValue(); // 得到跳转href

HttpGet httpget1 = new HttpGet(url);

response = httpclient.execute(httpget1);

// 登陆成功。。。hoho

}

entity = response.getEntity();

// System.out.println(response.getStatusLine());

if (entity != null) {

// System.out.println("Response content length: " + entity.getContentLength());

}

// 显示结果

BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), "UTF-8"));

String line = null;

while ((line = reader.readLine()) != null) {

// System.out.println(line);

}

//自动打卡

// 访问网站的子网页。

HttpPost httppost1 = new HttpPost(""); // 设置个人信息页面

httppost1.addHeader("Content-Type", "text/plain;charset=UTF-8");

httppost1.addHeader("Accept", "text/plain, */*");

httppost1.addHeader("X-Requested-With", "XMLHttpRequest");

httppost1.addHeader("Referer", "");

response = httpclient.execute(httppost1);

entity = response.getEntity();

// System.out.println(response.getStatusLine());

if(response.getStatusLine().toString().indexOf("HTTP/1.1 200 OK")=0){

ret = true;

}

if (entity != null) {

// System.out.println("Response content length: " + entity.getContentLength());

}

// 显示结果

reader = new BufferedReader(new InputStreamReader(entity.getContent(), "UTF-8"));

line = null;

while ((line = reader.readLine()) != null) {

System.out.println(line);

}

} catch (Exception e) {

} finally {

httpclient.getConnectionManager().shutdown();

}

return ret;

}

}

Q5: 怎么用java写网络爬虫将网页中的指定数据下载到本地excel文档中

mport java.io.InputStream;

        import java.net.*;

        public class HelloHttp {

        }

        接著就可以仿照下列范例建立HTTP连线：

        URL url = new URL("");

        HttpURLConnection http = (HttpURLConnection) url.openConnection();

        http.setRequestMethod("POST");

        InputStream input = http.getInputStream();

        http.disconnect();

    第1行建立一个URL物件，带入参数为想要建立HTTP连线的目的地，例如网站的网址。

    第2行建立一个HttpURLConnection物件，并利用URL的openConnection()来建立连线。

    第3行利用setRequestMethod()来设定连线的方式，一般分为POST及GET两种。

    第4行将连线取得的回应载入到一个InputStream中，然後就可以将InputStream的内容取出应用，以这个例子而言我们取得的会是网页的原始码。

    第5行用disconnect()将连线关闭。 

       将InputStream内容取出应用的范例如下：

        byte[] data = new byte[1024];

        int idx = input.read(data);

        String str = new String(data, 0, idx);

        System.out.println(str);

        input.close(); 

        针对 str 作 regular expression 处理 , 依照需求取得内容。

java的爬虫代码怎么写的介绍就聊到这里吧，感谢你花时间阅读本站内容，更多关于java的爬虫代码怎么写出来、java的爬虫代码怎么写的信息别忘了在本站进行查找喔。