正文
java下载网页源代码,java获取网页源代码
提示：扫一扫查出行【扫一扫了解最新限行尾号】
复制提示
Java访问指定URL并获取网页源代码
1．编写useSourceViewer 类的基本框架，该类仅包括无返回值的main ()方法，该方法从参数中获取URL，通过输入缓冲和输出缓冲将该URL 原码输出。
2．编写useSourceViewer 类，代码如下：
import java.net.*;
import java.io.*;
public class useSourceViewer
{
public static void main (String[] args)
{
if (args.length 0)
{
try
{
//读入URL
URL u = new URL(args[0]);
InputStream in = u.openStream( );
// 为增加性能存储输入流
in = new BufferedInputStream(in);
// 将输入流连接到阅读器
Reader r = new InputStreamReader(in);
int c;
while ((c = r.read( )) != -1)
{
System.out.print((char) c);
}
Object o = u.getContent( );
System.out.println("I got a " + o.getClass().getName( ));
}
catch (MalformedURLException e)
{
System.err.println(args[0] + " is not a parseable URL");
}
catch (IOException e)
{
System.err.println(e);
}
} // end if
} // end main
} // end SourceViewer}
相关问答
Q1: java中如何根据一个网址获得该网页的源代码，急求
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
public class HttpTest {
String urlString;
public static void main(String[] args) throws Exception {
HttpTest client = new HttpTest(网址);
client.run();
}
public HttpTest(String urlString) {
this.urlString = urlString;
}
public void run() throws Exception {
//生成一个URL对象
URL url = new URL(urlString);
//打开URL
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
//得到输入流，即获得了网页的内容
BufferedReader reader = new BufferedReader(new InputStreamReader(urlConnection
.getInputStream()));
String line;
// 读取输入流的数据，并显示
while ((line = reader.readLine()) != null){
System.out.println(line);
}
}
}
Q2: java中如何根据一个网址获得该网页的源代码？
package test;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
public class HttpTest {
private String u;
private String encoding;
public static void main(String[] args) throws Exception {
HttpTest client = new HttpTest("", "UTF-8");
client.run();
}
public HttpTest(String u, String encoding) {
this.u = u;
this.encoding = encoding;
}
public void run() throws Exception {
URL url = new URL(u);// 根据链接（字符串格式），生成一个URL对象
HttpURLConnection urlConnection = (HttpURLConnection) url
.openConnection();// 打开URL
BufferedReader reader = new BufferedReader(new InputStreamReader(
urlConnection.getInputStream(), encoding));// 得到输入流，即获得了网页的内容
String line; // 读取输入流的数据，并显示
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
}
根据具体问题类型，进行步骤拆解／原因原理分析／内容拓展等。
具体步骤如下：／导致这种情况的原因主要是……
Q3: 求一个Java实现自动下载的源代码
您好，这样的：
/**
*
*/
package com.b510.base.bean.install;
import java.io.File;
import java.io.FileWriter;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* @author hongten(hongtenzone@foxmail.com)
* @date 2013-2-24
*/
@SuppressWarnings("unchecked")
public class BeanUtils {
//公共部分
private static final String RT_1 = "\r\n";
private static final String RT_2 = RT_1+RT_1;
private static final String BLANK_1 =" ";
private static final String BLANK_4 =" ";
private static final String BLANK_8 =BLANK_4 + BLANK_4;
//注释部分
private static final String ANNOTATION_AUTHOR_PARAMTER = "@author ";
private static final String ANNOTATION_AUTHOR_NAME = "hongten(hongtenzone@foxmail.com)";
private static final String ANNOTATION_AUTHOR = ANNOTATION_AUTHOR_PARAMTER + ANNOTATION_AUTHOR_NAME;
private static final String ANNOTATION_DATE = "@date ";
private static final String ANNOTATION = "/**"+RT_1+BLANK_1+"*"+BLANK_1+ANNOTATION_AUTHOR +RT_1+BLANK_1+"*"+BLANK_1+ANNOTATION_DATE +getDate()+RT_1+BLANK_1+"*/"+RT_1;
//文件 地址
//private static final String BEAN_PATH = "com/b510/base/bean";
private static final String DAO_PATH = "com/b510/base/dao";
private static final String DAO_IMPL_PATH = "com/b510/base/dao/impl";
private static final String SERVICE_PATH = "com/b510/base/service";
private static final String SERVICE_IMPL_PATH = "com/b510/base/service/impl";
//包名
private static final String BEAN_URL = "com.b510.base.bean";
private static final String DAO_URL = "com.b510.base.dao";
private static final String DAO_IMPL_URL = "com.b510.base.dao.impl";
private static final String SERVICE_URL = "com.b510.base.service";
private static final String SERVICE_IMPL_URL = "com.b510.base.service.impl";
//基本类名称
private static final String BASE_DAO_NAME = DAO_URL + ".BaseDao";
private static final String ABSTRACT_BASE_DAO_IMPL_NAME = DAO_IMPL_URL + ".AbstractBaseDaoImpl";
private static final String BASE_SERVICE_NAME = SERVICE_URL + ".BaseService";
private static final String ABSTRACT_BASE_SERVICE_IMPL_NAME = SERVICE_IMPL_URL + ".AbstractBaseServiceImpl";
/**
* 创建bean的Daobr
*
* @param c
* @throws Exception
*/
public void createBeanDao(Class c) throws Exception {
String cName = c.getName();
String fileName = System.getProperty("user.dir") + "/src/" + DAO_PATH
+ "/" + getLastChar(cName) + "Dao.java";
File f = new File(fileName);
FileWriter fw = new FileWriter(f);
fw.write("package "+DAO_URL+";"+RT_2+ANNOTATION+"public interface " +
getLastChar(cName) + "Dao extends "+BASE_DAO_NAME+" " + cName + " {"+RT_2+"}");
fw.flush();
fw.close();
showInfo(fileName);
}
/**
* 创建bean的Dao的实现类
* @param c
* @throws Exception
*/
public void createBeanDaoImpl(Class c) throws Exception{
String cName = c.getName();
String fileName = System.getProperty("user.dir") + "/src/" + DAO_IMPL_PATH
+ "/" + getLastChar(cName) + "DaoImpl.java";
File f = new File(fileName);
FileWriter fw = new FileWriter(f);
fw.write("package "+DAO_IMPL_URL+";"+RT_2+ANNOTATION+"public class " +
getLastChar(cName) + "DaoImpl extends "+ABSTRACT_BASE_DAO_IMPL_NAME+"" +
cName + " implements "+DAO_URL+"."+getLastChar(cName)+"Dao{"+RT_2+"}");
fw.flush();
fw.close();
showInfo(fileName);
}
/**
* 创建bean的service
* @param c
* @throws Exception
*/
public void createBeanService(Class c) throws Exception{
String cName = c.getName();
String fileName = System.getProperty("user.dir") + "/src/" + SERVICE_PATH
+ "/" + getLastChar(cName) + "Service.java";
File f = new File(fileName);
FileWriter fw = new FileWriter(f);
fw.write("package "+SERVICE_URL+";"+RT_2+ANNOTATION+"public interface " +
getLastChar(cName) + "Service extends "+BASE_SERVICE_NAME+""+ cName +"{"+RT_2+"}");
fw.flush();
fw.close();
showInfo(fileName);
}
/**
* 创建bean的service的实现类
* @param c
* @throws Exception
*/
public void createBeanServiceImpl(Class c) throws Exception{
String cName = c.getName();
String fileName = System.getProperty("user.dir") + "/src/" + SERVICE_IMPL_PATH
+ "/" +getLastChar(cName)+"ServiceImpl.java";
File f = new File(fileName);
FileWriter fw = new FileWriter(f);
fw.write("package "+SERVICE_IMPL_URL+";"+RT_2+ANNOTATION+"public class "
+ getLastChar(cName) + "ServiceImpl extends "+ABSTRACT_BASE_SERVICE_IMPL_NAME+""+ cName
+ " implements "+SERVICE_URL+"."+getLastChar(cName)+"Service{"+RT_2+BLANK_4
+"private "+DAO_URL+"."+getLastChar(cName)+"Dao "+getLowercaseChar(getLastChar(cName))
+"Dao;"+RT_2+BLANK_4+"public void set"+getLastChar(cName)+"Dao("+DAO_URL+"."+getLastChar(cName)+"Dao "
+getLowercaseChar(getLastChar(cName))+"Dao){"+RT_1+BLANK_8+"this."+getLowercaseChar(getLastChar(cName))+"Dao = "
+getLowercaseChar(getLastChar(cName))+"Dao;"+RT_1+BLANK_4+"}"+RT_2+BLANK_4+"@Override"+RT_1+BLANK_4
+"public "+DAO_URL+"."+"BaseDao"+BEAN_URL+"."+getLastChar(cName)+" getBaseDao(){"+RT_1+BLANK_8
+"return "+getLowercaseChar(getLastChar(cName))+"Dao;"+RT_1+BLANK_4+"}"+RT_2+"}");
fw.flush();
fw.close();
showInfo(fileName);
}
/**
* 获取路径的最后面字符串br
* 如：br
* codestr = "com.b510.base.bean.User"/codebr
* code return "User";code
* @param str
* @return
*/
public String getLastChar(String str) {
if ((str != null) (str.length() 0)) {
int dot = str.lastIndexOf('.');
if ((dot -1) (dot (str.length() - 1))) {
return str.substring(dot + 1);
}
}
return str;
}
/**
* 把第一个字母变为小写br
* 如：br
* codestr = "UserDao";/codebr
* codereturn "userDao";/code
* @param str
* @return
*/
public String getLowercaseChar(String str){
return str.substring(0,1).toLowerCase()+str.substring(1);
}
/**
* 显示信息
* @param info
*/
public void showInfo(String info){
System.out.println("创建文件："+ info+ "成功！");
}
/**
* 获取系统时间
* @return
*/
public static String getDate(){
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
return simpleDateFormat.format(new Date());
}
}
Q4: java 获取网页源代码---有效防止乱码
前段时间做过这类功能，如何有效防止乱码，我们必须先知道一个网页的编码方式，是utf-8,还是gbk。
1.HttpURLConnection.getContentType();直接读取，效率高，但有很多时候读不到。只是text/html就完事了，没有charset.
2.使用第三方的HttpClient,执行效率较高。但读取网页头header也只适用部分站，很多网站服务段不设置，结果就读成了null.
3.最没有效率的判断方法就是使用inputStreamReader先把正页的html源码读取出来，之后截取charset后面编码。得到编码之后重新再读取一遍。但是效率很低。
做个总结：
/**
* 取得页面编码
*
* @param url
* @return
*/
public String getCharset(String url) throws Exception {
// log.info("进入读页面的关键词:" + keyword);
String charset = "";
int c;
HttpURLConnection httpurlcon = null;
// log.info("url:"+url);
// log.info("charset:"+charset);
log.info("url:" + url);
URL httpurl = new URL(url);
// System.out.println(url+str);
httpurlcon = (HttpURLConnection) httpurl.openConnection();
// google需要身份
httpurlcon.setRequestProperty("User-agent", "Mozilla/4.0");
charset = httpurlcon.getContentType();
log.info("charset1:" + charset);
// 如果可以找到
if (charset.indexOf("charset=") != -1)
charset = charset.substring(charset.indexOf("charset=")
+ "charset=".length(), charset.length());
// 否则读取response.Header头
else {
charset = this.getContentCharset();
log.info("charset2:" + charset);
}
// 如果charset还是为空,那么直接读网页来截取
if (charset == null) {
charset = this.readPageCharset(url);
log.info("charset31:" + charset);
}
return charset;
}
Q5: 如何用java中HTMLPraser提取网页源代码中的某些链接
使用正则表达式去匹配就行了。
第一步：下载你需要分析的网页的源码
第二步：在程序中使用正则表达式去匹配源码，保存匹配成功的链接地址就行。
可以在网上搜一下匹配网址的正则表达式
