http://xiangai.taobao.com
http://shop148612228.taobao.com

java实现大文件下载(http方式)

java实现大文件下载,基于http方式,控件神马的就不说了。
思路:下载文件无非要读取文件然后写文件,主要这两个步骤,主要难点:
1.读文件,就是硬盘到内存的过程,由于jdk内存限制,不能读的太大。
2.写文件,就是响应到浏览器端的过程,http协议是短链接,如果写文件太慢,时间过久,会造成浏览器死掉。

知识点:

1.org.apache.http.impl.client.CloseableHttpClient  模拟httpClient客户端发送http请求,可以控制到请求文件的字节位置。
2.BufferedInputStream都熟悉,用它接受请求来的流信息缓存。
3.RandomAccessFile文件随机类,可以向文件写入指定位置的流信息。
  • 1
  • 2
  • 3

基于以上信息,我的实现思路就是首先判断下载文件大小,配合多线程分割定制http请求数量和请求内容,响应到写入到RandomAccessFile指定位置中。在俗点就是大的http分割成一个个小的http请求,相当于每次请求一个网页。

废话不说,上代码。

DownLoadManagerTest类:

package xxxx;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

/**
 * 
 * 文件下载管理类
 */
@RunWith(SpringJUnit4ClassRunner.class)
@ActiveProfiles("test")
@ContextConfiguration(locations={"classpath:test/applicationContext.xml"})
public class DownLoadManagerTest extends AbstractTransactionalJUnit4SpringContextTests{
	
	private static final Logger LOGGER = LoggerFactory.getLogger(DownLoadManagerTest.class);

	/**
	 * 
	 * 每个线程下载的字节数
	 */
	private long unitSize = 1000 * 1024;
	
	@Autowired
	private TaskExecutor taskExecutor;
	
	private CloseableHttpClient httpClient;
	
	private Long starttimes;
	
	private Long endtimes;
    
    @Before
    public void setUp() throws Exception{
    	starttimes = System.currentTimeMillis();
        System.out.println("测试开始....");
    }
    
    @After
    public void tearDown() throws Exception{
    	endtimes = System.currentTimeMillis();
        System.out.println("测试结束!!");
        System.out.println("********************");
        System.out.println("下载总耗时:"+(endtimes-starttimes)/1000+"s");
        System.out.println("********************");
    }

	public DownLoadManagerTest() {
		System.out.println("初始化测试类....");
		PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
		cm.setMaxTotal(100);
		httpClient = HttpClients.custom().setConnectionManager(cm).build();
		
	}

	/**
	 * 
	 * 启动多个线程下载文件
	 */
	@Test
	public void  doDownload() throws IOException {

		String remoteFileUrl="http://{host}:{port}/{project}/xx.xml";
		String localPath="E://test//";
		
		String fileName = new URL(remoteFileUrl).getFile();

		System.out.println("远程文件名称:"+fileName);
		fileName = fileName.substring(fileName.lastIndexOf("/") + 1,
				fileName.length()).replace("%20", " ");
		System.out.println("本地文件名称:"+fileName);
		long fileSize = this.getRemoteFileSize(remoteFileUrl);

		this.createFile(localPath+System.currentTimeMillis()+fileName, fileSize);

		Long threadCount = (fileSize/unitSize)+(fileSize % unitSize!=0?1:0);
		long offset = 0;
		
		CountDownLatch end = new CountDownLatch(threadCount.intValue());
		// 如果远程文件尺寸小于等于unitSize
		if (fileSize <= unitSize) {

			DownloadThreadTest downloadThread = new DownloadThreadTest(remoteFileUrl,
					localPath+fileName, offset, fileSize,end,httpClient);

			taskExecutor.execute(downloadThread);

		} else {
			// 如果远程文件尺寸大于unitSize
			for (int i = 1; i < threadCount; i++) {
				DownloadThreadTest downloadThread = new DownloadThreadTest(
				remoteFileUrl, localPath+fileName, offset, unitSize,end,httpClient);
				taskExecutor.execute(downloadThread);
				offset = offset + unitSize;

			}
			// 如果不能整除,则需要再创建一个线程下载剩余字节
			if (fileSize % unitSize != 0) {
				DownloadThreadTest downloadThread = new DownloadThreadTest(remoteFileUrl, localPath+fileName, offset, fileSize - unitSize * (threadCount-1),end,httpClient);
				taskExecutor.execute(downloadThread);
			}
		}
		try {
			end.await();
		} catch (InterruptedException e) {
			LOGGER.error("DownLoadManager exception msg:{}",ExceptionUtils.getFullStackTrace(e));
			e.printStackTrace();
		}
//		System.out.println("111111");
		LOGGER.debug("下载完成!{} ",localPath+fileName);
		//return localPath+fileName;
	}

	/**
	 * 
	 * 获取远程文件尺寸
	 */
	private long getRemoteFileSize(String remoteFileUrl) throws IOException {
		long fileSize = 0;
		HttpURLConnection httpConnection = (HttpURLConnection) new URL(
		remoteFileUrl).openConnection();
		httpConnection.setRequestMethod("HEAD");
		int responseCode = httpConnection.getResponseCode();
		if (responseCode >= 400) {
			LOGGER.debug("Web服务器响应错误!");
			return 0;
		}

		String sHeader;
		for (int i = 1;; i++) {
			sHeader = httpConnection.getHeaderFieldKey(i);
			if (sHeader != null && sHeader.equals("Content-Length")) {
				System.out.println("文件大小ContentLength:"
						+ httpConnection.getContentLength());
				fileSize = Long.parseLong(httpConnection
						.getHeaderField(sHeader));
				break;
			}
		}
		return fileSize;
	}

	/**
	 * 
	 * 创建指定大小的文件
	 */
	private void createFile(String fileName, long fileSize) throws IOException {
		File newFile = new File(fileName);
		RandomAccessFile raf = new RandomAccessFile(newFile, "rw");
		raf.setLength(fileSize);
		raf.close();
	}

	public TaskExecutor getTaskExecutor() {
		return taskExecutor;
	}

	public void setTaskExecutor(TaskExecutor taskExecutor) {
		this.taskExecutor = taskExecutor;
	}
}
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
  • 70
  • 71
  • 72
  • 73
  • 74
  • 75
  • 76
  • 77
  • 78
  • 79
  • 80
  • 81
  • 82
  • 83
  • 84
  • 85
  • 86
  • 87
  • 88
  • 89
  • 90
  • 91
  • 92
  • 93
  • 94
  • 95
  • 96
  • 97
  • 98
  • 99
  • 100
  • 101
  • 102
  • 103
  • 104
  • 105
  • 106
  • 107
  • 108
  • 109
  • 110
  • 111
  • 112
  • 113
  • 114
  • 115
  • 116
  • 117
  • 118
  • 119
  • 120
  • 121
  • 122
  • 123
  • 124
  • 125
  • 126
  • 127
  • 128
  • 129
  • 130
  • 131
  • 132
  • 133
  • 134
  • 135
  • 136
  • 137
  • 138
  • 139
  • 140
  • 141
  • 142
  • 143
  • 144
  • 145
  • 146
  • 147
  • 148
  • 149
  • 150
  • 151
  • 152
  • 153
  • 154
  • 155
  • 156
  • 157
  • 158
  • 159
  • 160
  • 161
  • 162
  • 163
  • 164
  • 165
  • 166
  • 167
  • 168
  • 169
  • 170
  • 171
  • 172
  • 173
  • 174
  • 175
  • 176
  • 177
  • 178
  • 179
  • 180

DownloadThreadTest类:

package xxxx;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.concurrent.CountDownLatch;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 
 * 负责文件下载的类
 */
public class DownloadThreadTest extends Thread {

	private static final Logger LOGGER = LoggerFactory
			.getLogger(DownloadThreadTest.class);

	/**
	 * 
	 * 待下载的文件
	 */
	private String url = null;

	/**
	 * 
	 * 本地文件名
	 */
	private String fileName = null;

	/**
	 * 
	 * 偏移量
	 */
	private long offset = 0;

	/**
	 * 
	 * 分配给本线程的下载字节数
	 */
	private long length = 0;
	private CountDownLatch end;
	private CloseableHttpClient httpClient;
	private HttpContext context;

	/**
	 * 
	 * @param url 下载文件地址
	 * 
	 * @param fileName 另存文件名
	 * 
	 * @param offset  本线程下载偏移量
	 * 
	 * @param length  本线程下载长度
	 * 
	 * @author Angus.wang
	 * 
	 * */
	public DownloadThreadTest(String url, String file, long offset, long length,
			CountDownLatch end, CloseableHttpClient httpClient) {
		this.url = url;
		this.fileName = file;
		this.offset = offset;
		this.length = length;
		this.end = end;
		this.httpClient = httpClient;
		this.context = new BasicHttpContext();
		LOGGER.debug("偏移量=" + offset + ";字节数=" + length);
	}

	public void run() {
		try {
			HttpGet httpGet = new HttpGet(this.url);
			httpGet.addHeader("Range", "bytes=" + this.offset + "-"
					+ (this.offset + this.length - 1));
			CloseableHttpResponse response = httpClient.execute(httpGet,
					context);
			BufferedInputStream bis = new BufferedInputStream(response
					.getEntity().getContent());
			byte[] buff = new byte[1024];
			int bytesRead;
			File newFile = new File(fileName);
			RandomAccessFile raf = new RandomAccessFile(newFile, "rw");
			while ((bytesRead = bis.read(buff, 0, buff.length)) != -1) {
				raf.seek(this.offset);
				raf.write(buff, 0, bytesRead);
				this.offset = this.offset + bytesRead;
			}
			raf.close();
			bis.close();
		} catch (ClientProtocolException e) {
			LOGGER.error("DownloadThread exception msg:{}",ExceptionUtils.getFullStackTrace(e));
		} catch (IOException e) {
			LOGGER.error("DownloadThread exception msg:{}",ExceptionUtils.getFullStackTrace(e));
		} finally {
			end.countDown();
			LOGGER.info(end.getCount() + " is go on!");
			System.out.println(end.getCount() + " is go on!");
		}
	}
}
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
  • 70
  • 71
  • 72
  • 73
  • 74
  • 75
  • 76
  • 77
  • 78
  • 79
  • 80
  • 81
  • 82
  • 83
  • 84
  • 85
  • 86
  • 87
  • 88
  • 89
  • 90
  • 91
  • 92
  • 93
  • 94
  • 95
  • 96
  • 97
  • 98
  • 99
  • 100
  • 101
  • 102
  • 103
  • 104
  • 105
  • 106
  • 107
  • 108
  • 109

application.xml

<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
		<!-- 线程池活跃的线程数 -->
		<property name="corePoolSize" value="5" />
		<!-- 线程池最大活跃的线程数 -->
		<property name="maxPoolSize" value="10" />
		<!-- 队列的最大容量 -->
		<property name="queueCapacity" value="600" />
	</bean>
	<bean id="downLoadManager"
		class="xx.DownLoadManagerTest">
		<property name="taskExecutor" ref="taskExecutor" />
	</bean>
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12

测试运行,500M,我这网速得半个小时左右。要想下载更大的文件,只要jdk内存够大,就无限更改队列最大容量吧。

如果不同意见,欢迎各位大神指正。

转载:https://my.oschina.net/zmf/blog/336961

posted @ 2021-12-08 16:00  万事俱备就差个程序员  阅读(2822)  评论(0编辑  收藏  举报

http://xiangai.taobao.com
http://shop148612228.taobao.com
如果您觉得对您有帮助.领个红包吧.谢谢.
支付宝红包
微信打赏 支付宝打赏