java抓取网页内容 - JAVA

public static void main(String[] args) {
Test t = new Test();
String content = t.getHtmlContent("http://www.taobao.com");
//content = content.replaceAll("(
)+ ", "\n");// 转化换行
//content = content.replaceAll("

", "");// 去图片注释
System.out.println(content);
System.out.println(t.getTitle(content));
List a = t.getNews(content);
List news = new ArrayList();
for (String s : a) {
news.add(s.replaceAll("<.* >", ""));
}
System.out.println(news);
//…… 获取js、css等操作省略
}
}

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
*
* @author yaohucaizi
*/
public class Test {

/**
* 读取网页全部内容
*/
public String getHtmlContent(String htmlurl) {
URL url;
String temp;
StringBuffer sb = new StringBuffer();
try {
url = new URL(htmlurl);
BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "gbk"));// 读取网页全部内容
while ((temp = in.readLine()) != null) {
sb.append(temp);
}
in.close();
} catch (final MalformedURLException me) {
System.out.println("你输入的URL格式有问题!");
me.getMessage();
} catch (final IOException e) {
e.printStackTrace();
}
return sb.toString();
}

/**
*
* @param s
* @return 获得网页标题
*/
public String getTitle(String s) {
String regex;
String title = "";
List list = new ArrayList();
regex = ".* ";
Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
title = title + list.get(i);
}
return outTag(title);
}

/**
*
* @param s
* @return 获得链接
*/
public List getLink(String s) {
String regex;
List list = new ArrayList();
regex = "]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.* )";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
return list;
}

/**
*
* @param s
* @return 获得脚本代码
*/
public List getScript(String s) {
String regex;
List list = new ArrayList();
regex = "";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
return list;
}

public List getNews(String s) {
String regex = "";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(s);
List list = new ArrayList();
while (ma.find()) {
list.add(ma.group());
}
return list;
}

/**
*
* @param s
* @return 获得CSS
*/
public List getCSS(String s) {
String regex;
List list = new ArrayList();
regex = "";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);

java抓取网页内容 (二)