Httpclient 下载资源

HttpClient 下载网络资源

所需commons-httpclient-3.0.1.jar及其依赖包

package cn.test.softcrawltool.utils.net;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import cn.yicha.softcrawltool.utils.CommonUtils;
import static cn.test.softcrawltool.utils.consts.SoftCrawlConst.*;

/**
* 功能: 软件资源下载
*
* @author liaoyq
*
*/
public class DownloadUtil {

private static final Logger LOG = Logger.getLogger(DownloadUtil.class);

private static final int BUFFER_SIZE = 10240; //10K

public static boolean downloadBinaryResource(String url, String referer, String rootpath) {
File rootDir = CommonUtils.createFileDirectory(rootpath);

HttpClient httpClient = getHttpClient();
GetMethod getMethod = getMethod(url, referer);

InputStream in = null;
DataOutputStream dos = null;
try {
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode == HttpStatus.SC_OK) {
String newurl = getMethod.getResponseHeader("Content-Location").getValue();
String filename = getFilename(newurl);
in = getMethod.getResponseBodyAsStream();
if (in != null) {
dos = new DataOutputStream(new FileOutputStream(new File(rootDir, filename)));
byte[] buffer = new byte[BUFFER_SIZE];
int bytes = 0;
while ((bytes = in.read(buffer)) != -1)
dos.write(buffer, 0, bytes);
return true;
}
} else {
LOG.error("Method failed: " + getMethod.getStatusLine());
}
} catch (Exception e) {
LOG.error("download soft resource failed!" + url, e);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException e) {
}
}
if (dos != null) {
try {
dos.close();
} catch (IOException e) {
}
}
getMethod.releaseConnection();
}
return false;
}

public static String downloadPage(String url, String pageEncoding) {
StringBuffer content = new StringBuffer(10240);
HttpClient httpClient = getHttpClient();
GetMethod getMethod = getMethod(url, null);

InputStream in = null;
try {
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode == HttpStatus.SC_OK) {
in = getMethod.getResponseBodyAsStream();
if (in != null) {
BufferedReader br = new BufferedReader(new InputStreamReader(in, pageEncoding));
String line = null;
while ((line = br.readLine()) != null) {
content.append(line + "\n");
}
return content.toString();
}
}
} catch (Exception e) {
LOG.error("download page content failed!" + url, e);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException e) {
}
}
getMethod.releaseConnection();
}
return null;
}

private static HttpClient getHttpClient() {
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
HttpClient httpClient = new HttpClient(connectionManager);
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(
ConnectionTimeout);
httpClient.getHttpConnectionManager().getParams().setSoTimeout(
SoTimeout);
return httpClient;
}

private static GetMethod getMethod(String url, String refererUrl) {
GetMethod getMethod = new GetMethod(url);
getMethod.setFollowRedirects(true);// 自动重定向
if (!StringUtils.isEmpty(refererUrl))
getMethod.setRequestHeader(Referer, refererUrl);// 解决不能下载问题
getMethod.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
getMethod.setRequestHeader(COOKIE, COOKIE_VALUE);
getMethod.setRequestHeader(USER_AGENT, USER_AGENT_VALUE);
getMethod.setRequestHeader(Connection, Connection_VALUE);
return getMethod;
}

private static String getFilename(String url) {
return url.substring(url.lastIndexOf("/") + 1);
}
}