前言:
時下互聯網第一波的浪潮已消逝,隨著而來的基于萬千數據的物聯網時代,因而數據成為企業的重要戰略資源之一。基于數據抓取技術,本文介紹了java相關抓取工具,并附上demo源碼供感興趣的朋友測試!
1)JDK自帶HTTP連接,獲取頁面或Json
2) JDK自帶URL連接,獲取頁面或Json
3)HttpClient Get工具,獲取頁面或Json
4)commons-io工具,獲取頁面或Json
5) Jsoup工具(通常用于html字段解析),獲取頁面,非Json返回格式】
--------------------------------------------------------------------------------
完整代碼:
package com.yeezhao.common.http;import java.io.BufferedReader;import java.io.InputStream;import java.io.InputStreamReader;import java.net.HttpURLConnection;import java.net.URL;import org.apache.commons.httpclient.HttpClient;import org.apache.commons.httpclient.HttpMethod;import org.apache.commons.httpclient.methods.GetMethod;import org.apache.commons.io.IOUtils;import org.jsoup.Jsoup;/** * http工具對比 * * @author Administrator -> junhong * * 2016年12月27日 */public class HttpFetchUtil { /** * 獲取訪問的狀態碼 * @param request * @return * @throws Exception */ public static int getResponseCode(String request) throws Exception { URL url = new URL(request); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); return conn.getResponseCode(); } /** * 1)JDK自帶HTTP連接,獲取頁面或Json * @param request * @param charset * @return * @throws Exception */ public static String JDKFetch(String request, String charset) throws Exception { URL url = new URL(request); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); //模擬瀏覽器參數 conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"); if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { InputStream input = conn.getInputStream(); StringBuffer sb = new StringBuffer(); BufferedReader reader = new BufferedReader(new InputStreamReader(input, charset)); String s; while ((s = reader.readLine()) != null) { sb.append(s + "/n"); } input.close(); conn.disconnect(); return sb.toString(); } return ""; } /** * 2) JDK自帶URL連接,獲取頁面或Json * @param request * @param charset * @return * @throws Exception */ public static String URLFetch(String request, String charset) throws Exception { URL url = new URL(request); return IOUtils.toString(url.openStream()); } /** * 3)HttpClient Get工具,獲取頁面或Json * @param url * @param charset * @return * @throws Exception */ public static String httpClientFetch(String url, String charset) throws Exception { // GET HttpClient httpClient = new HttpClient(); httpClient.getParams().setContentCharset(charset); HttpMethod method = new GetMethod(url); httpClient.executeMethod(method); return method.getResponseBodyAsString(); } /** * 4)commons-io工具,獲取頁面或Json * @param url * @param charset * @return * @throws Exception */ public static String commonsIOFetch(String url, String charset) throws Exception { return IOUtils.toString(new URL(url), charset); } /** * 5) Jsoup工具(通常用于html字段解析),獲取頁面,非Json返回格式 * @param url * @return * @throws Exception */ public static String jsoupFetch(String url) throws Exception { return Jsoup.parse(new URL(url), 2 * 1000).html(); }}測試代碼:
package com.yeezhao.common.http;import org.junit.After;import org.junit.Before;import org.junit.Test;/** * 測試類 * 3個測試鏈接: * 1)百科網頁 * 2)瀏覽器模擬獲取接口數據 * 3)獲取普通接口數據 * @author Administrator -> junhong * * 2016年12月27日 */public class HttpFetchUtilTest { String seeds[] = {"http://baike.baidu.com/view/1.htm","http://m.ximalaya.com/tracks/26096131.json","http://remyapi.yeezhao.com/api/query?wd=%E5%91%A8%E6%98%9F%E9%A9%B0%E7%9A%84%E7%94%B5%E5%BD%B1"}; final static String DEFAULT_CHARSET = "UTF-8"; @Before public void setUp() throws Exception { } @After public void tearDown() throws Exception { System.out.println("--- down ---"); } @Test public void testGetResponseCode() throws Exception{ for(String seed:seeds){ int responseCode = HttpFetchUtil.getResponseCode(seed); System.out.println("ret="+responseCode); } } @Test public void testJDKFetch() throws Exception{ for(String seed:seeds){ String ret = HttpFetchUtil.JDKFetch(seed, DEFAULT_CHARSET); System.out.println("ret="+ret); } } @Test public void testURLFetch() throws Exception{ for(String seed:seeds){ String ret = HttpFetchUtil.URLFetch(seed, DEFAULT_CHARSET); System.out.println("ret="+ret); } } @Test public void testHttpClientFetch()throws Exception { for(String seed:seeds){ String ret = HttpFetchUtil.httpClientFetch(seed, DEFAULT_CHARSET); System.out.println("ret="+ret); } } @Test public void testCommonsIOFetch()throws Exception { for(String seed:seeds){ String ret = HttpFetchUtil.commonsIOFetch(seed, DEFAULT_CHARSET); System.out.println("ret="+ret); } } @Test public void testJsoupFetch() throws Exception{ for(String seed:seeds){ String ret = HttpFetchUtil.jsoupFetch(seed); System.out.println("ret="+ret); } }}附:相關jar依賴
...<dependency><groupId>org.jsoup</groupId><artifactId>jsoup</artifactId><version>1.7.3</version></dependency><dependency><groupId>commons-httpclient</groupId><artifactId>commons-httpclient</artifactId><version>3.1</version></dependency><dependency><groupId>commons-io</groupId><artifactId>commons-io</artifactId><version>2.4</version></dependency>...
以上就是本文的全部內容,希望對大家的學習有所幫助,也希望大家多多支持武林網。
新聞熱點
疑難解答