多线程断点续传 -

a38876399

浏览: 147859 次
性别:
来自: 广州

最近访客更多访客>>

xiaobai991

u011630130

yuenxm

liu_3378139

博主相关

博客

微博

相册

留言

关于我

文章分类

全部博客 (68)

社区版块

存档分类

多线程断点续传

多线程 Java OS Ubuntu Linux

以前看过一个前辈写的断点续传的文章，记得当时没看懂，就扔那了。昨天翻了出来，自己也仿照写了一个，不过感觉没人家写的好，有点乱，希望大家能来批评、指正，给点意见！

功能很简单，就是启动多个线程分别从给定的地址下载数据，用RandomAccessFile写到目标文件。实现思路是：

1、获得连接的长度（即要下载的文件大小），除以设定的线程数，即得到每个线程要下载的大小。

2、记录临时文件，文件中记录每个线程的编号（id），该线程要下载的起始位置、终止位置和当前位置（当前位置在首次下载时与起始位置相同）。

3、启动具体执行下载任务的线程，并等待其结束。

4、下载完成，删除临时文件。

代码如下:

主线程及测试的main方法

package com.why.download.test;  
  
import java.io.DataOutputStream;  
import java.io.File;  
import java.io.FileOutputStream;  
import java.io.IOException;  
import java.math.BigDecimal;  
import java.net.HttpURLConnection;  
import java.net.URL;  
import java.net.URLConnection;  
import java.util.UUID;  
import java.util.concurrent.CountDownLatch;  
import java.util.concurrent.ExecutorService;  
import java.util.concurrent.Executors;  
  
/** 
 *  
 * @author why 
 * 
 */  
public class DownLoad {  
    //文件目录、文件名  
    public String fileDir = "E:/MyDownLoad";  
    public String fileName;  
    //超时重连时间  
    public long reconnectTime = 5;  
    //线程数  
    private int poolSize = 5;  
    //每个线程的缓冲区大小  
    public int bufferSize = 1024;  
    //url地址  
    private String urlLocation = null;  
      
    public DownLoad(){}  
    public DownLoad(String url){  
        this.urlLocation = url;  
    }  
    public void downLoad(){  
        if(this.urlLocation == null || "".equals(this.urlLocation))return;  
        downLoad(this.urlLocation);  
    }  
    public void downLoad(String urlLocation){  
        File file = null;  
        File tempFile = null;  
        CountDownLatch latch;  
        URL url = null;  
        ExecutorService pool = Executors.newCachedThreadPool();  
        long contentLength = 0;  
        long threadLength = 0;  
        try {  
            //如果未指定名称，则从url中获得下载的文件格式与名字  
            if(fileName == null || "".equals(fileName)){  
                this.fileName = urlLocation.substring(urlLocation.lastIndexOf("/") + 1,  
                        urlLocation.lastIndexOf("?") > 0 ? urlLocation.lastIndexOf("?")  
                                : urlLocation.length());  
                if ("".equalsIgnoreCase(this.fileName)) {  
                    this.fileName = UUID.randomUUID().toString();  
                }  
            }  
            new File(fileDir).mkdirs();  
            file = new File(fileDir + File.separator + fileName);  
            tempFile = new File(fileDir + File.separator + fileName + "_temp");  
              
            url = new URL(urlLocation);  
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();  
            setHeader(conn);  
            //得到content的长度  
            contentLength = conn.getContentLength();  
              
            System.out.println("total length=" + contentLength);  
              
            //把context分为poolSize段，计算每段的长度。  
//          threadLength = contentLength / this.poolSize;  
            BigDecimal b1 = new BigDecimal(Double.toString(contentLength));  
            BigDecimal b2 = new BigDecimal(Double.toString(this.poolSize));  
            threadLength = b1.divide(b2, 0, BigDecimal.ROUND_HALF_UP).longValue();  
              
            if(file.exists() && tempFile.exists()){  
                //如果文件已存在，根据临时文件中记载的线程数量，继续上次的任务  
                latch = new CountDownLatch((int)tempFile.length()/28);  
                for(int i=0;i<tempFile.length()/28;i++){  
                    pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));  
                }  
            }else{  
                //如果下载的目标文件不存在，则创建新文件  
                latch = new CountDownLatch(poolSize);  
                file.createNewFile();  
                tempFile.createNewFile();  
                DataOutputStream os = new DataOutputStream(new FileOutputStream(tempFile));  
                for(int i=0;i<this.poolSize;i++){  
                    os.writeInt(i+1);  
                    os.writeLong(i*threadLength);  
                    if(i==this.poolSize-1){//最后一个线程的结束位置应为文件末端  
                        os.writeLong(contentLength);  
                    }else{  
                        os.writeLong((i+1)*threadLength);  
                    }  
                    os.writeLong(i*threadLength);  
                    pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));  
                }  
                os.close();  
            }  
            //等待下载任务完成  
            latch.await();  
            //删除临时文件  
            tempFile.delete();  
        } catch (IOException e) {  
            e.printStackTrace();  
        } catch (InterruptedException e) {  
            e.printStackTrace();  
        } finally{  
            pool.shutdown();   
        }  
    }  
      
    private void setHeader(URLConnection conn) {  
        conn.setRequestProperty("User-Agent",  
                        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");  
        conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");  
        conn.setRequestProperty("Accept-Encoding", "aa");  
        conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");  
        conn.setRequestProperty("Keep-Alive", "300");  
        conn.setRequestProperty("Connection", "keep-alive");  
        conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");  
        conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");  
        conn.setRequestProperty("Cache-Control", "max-age=0");  
        conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");  
    }  
  
      
    public String getFileDir() {  
        return fileDir;  
    }  
    public void setFileDir(String fileDir) {  
        this.fileDir = fileDir;  
    }  
    public String getFileName() {  
        return fileName;  
    }  
    public void setFileName(String fileName) {  
        this.fileName = fileName;  
    }  
    public long getReconnectTime() {  
        return reconnectTime;  
    }  
    public void setReconnectTime(long reconnectTime) {  
        this.reconnectTime = reconnectTime;  
    }  
    public int getPoolSize() {  
        return poolSize;  
    }  
    public void setPoolSize(int poolSize) {  
        this.poolSize = poolSize;  
    }  
    public int getBufferSize() {  
        return bufferSize;  
    }  
    public void setBufferSize(int bufferSize) {  
        this.bufferSize = bufferSize;  
    }  
    /** 
     * @param args 
     */  
    public static void main(String[] args) {  
        DownLoad dl = new DownLoad();  
        dl.setFileDir("E:/MyDownLoad/music/");  
        dl.setFileName("大笑江湖.mp3");  
        dl.setPoolSize(20);  
        long beginTime = System.currentTimeMillis();  
        dl.downLoad("http://mh.163k.com/UploadFile/video/2010/12-13/201012131213448942190.mp3");  
        long endTime = System.currentTimeMillis();  
        BigDecimal b1 = new BigDecimal(endTime - beginTime);  
        BigDecimal b2 = new BigDecimal(1000);  
        double cost = b1.divide(b2, 2, BigDecimal.ROUND_HALF_UP).doubleValue();  
        System.out.println("Time cost:" + cost + "s");  
    }  
  
}

执行下载任务的线程：

package com.why.download.test;  
  
import java.io.File;  
import java.io.FileNotFoundException;  
import java.io.IOException;  
import java.io.InputStream;  
import java.io.RandomAccessFile;  
import java.net.HttpURLConnection;  
import java.net.URL;  
import java.net.URLConnection;  
import java.util.concurrent.Callable;  
import java.util.concurrent.CountDownLatch;  
import java.util.concurrent.TimeUnit;  
  
/** 
 *  
 * @author why 
 * 
 */  
public class DownLoadTask implements Callable<String>{  
    //超时重连时间  
    private long reconnectTime = 5;  
    //缓冲区大小  
    private int bufferSize = 1024;  
      
    private CountDownLatch latch;  
    private RandomAccessFile file = null;  
    private RandomAccessFile tempFile = null;  
    private URL url = null;  
    private int id;  
    private long startPosition;  
    private long endPosition;  
    private long currentPosition ;  
      
    public DownLoadTask(File file,File tempFile,URL url,int id,CountDownLatch latch,long reconnectTime,int bufferSize){  
        try {  
            this.file = new RandomAccessFile(file, "rw");  
            this.tempFile = new RandomAccessFile(tempFile, "rw");  
        } catch (FileNotFoundException e) {  
            e.printStackTrace();  
        }  
        this.url = url;  
        this.id = id;  
        this.latch = latch;  
    }  
      
    public String call(){  
          
        try {  
            tempFile.seek((id-1)*28);  
            tempFile.readInt();  
            this.startPosition = tempFile.readLong();  
            this.endPosition = tempFile.readLong();  
            this.currentPosition = tempFile.readLong();  
        } catch (IOException e) {  
            e.printStackTrace();  
        }  
          
        System.out.println("Thread " + id + " begin!");  
          
        HttpURLConnection conn = null;  
        InputStream inputStream = null;  
  
        while(true){  
            try {  
                tempFile.seek(id*28 - 8);  
                // 打开URLConnection  
                conn = (HttpURLConnection) this.url.openConnection();  
                setHeader(conn);  
                // 设置连接超时时间为10000ms  
                conn.setConnectTimeout(10000);  
                // 设置读取数据超时时间为10000ms  
                conn.setReadTimeout(10000);  
  
                if (currentPosition < endPosition) {  
                    // 设置下载数据的起止区间  
                    conn.setRequestProperty("Range", "bytes=" + currentPosition + "-" + endPosition);  
                      
                    System.out.println("Thread " + id + " startPosition=" + startPosition   
                            + ",endPosition=" + endPosition + ",currentPosition=" + currentPosition);  
  
                    file.seek(currentPosition);  
  
                    // 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK  
                    // 如果不是以上两种状态，把status改为STATUS_HTTPSTATUS_ERROR  
                    if (conn.getResponseCode() != HttpURLConnection.HTTP_OK  
                            && conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) {  
                        System.out.println("Thread " + id + ": code = " + conn.getResponseCode() + ", status = " + conn.getResponseMessage());  
                        file.close();  
                        conn.disconnect();  
                        System.out.println("Thread " + id + " finished.");  
                        break;  
                    }  
  
                    inputStream = conn.getInputStream();  
                    int len = 0;  
                    byte[] b = new byte[bufferSize];  
                    while ((len = inputStream.read(b)) != -1) {  
                        file.write(b, 0, len);  
  
                        currentPosition += len;  
                        // set tempFile now position  
                        tempFile.seek(id*28 - 8);  
                        tempFile.writeLong(currentPosition);  
                    }  
  
                    file.close();  
                    tempFile.close();  
                    inputStream.close();  
                    conn.disconnect();  
                }  
  
                System.out.println("Thread " + id + " finished.");  
                break;  
            } catch (IOException e) {  
                try {  
                    TimeUnit.SECONDS.sleep(getReconnectTime());  
                } catch (InterruptedException e1) {  
                    e1.printStackTrace();  
                }  
                continue;  
            }  
        }  
        latch.countDown();  
        return "finish";  
    }  
      
    private void setHeader(URLConnection conn) {  
        conn.setRequestProperty("User-Agent",  
                        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");  
        conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");  
        conn.setRequestProperty("Accept-Encoding", "aa");  
        conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");  
        conn.setRequestProperty("Keep-Alive", "300");  
        conn.setRequestProperty("Connection", "keep-alive");  
        conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");  
        conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");  
        conn.setRequestProperty("Cache-Control", "max-age=0");  
        conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");  
    }  
  
    public long getReconnectTime() {  
        return reconnectTime;  
    }  
  
    public void setReconnectTime(long reconnectTime) {  
        this.reconnectTime = reconnectTime;  
    }  
  
    public int getBufferSize() {  
        return bufferSize;  
    }  
  
    public void setBufferSize(int bufferSize) {  
        this.bufferSize = bufferSize;  
    }  
  
      
}

最近一直在测试，写文档，好久没敲代码了。手有点痒，写着玩的，代码写的不是很工整，纯属娱乐！

摘自：http://www.iteye.com/topic/869109