java实现大文件下载(http方式)
java实现大文件下载，基于http方式，控件神马的就不说了。
思路：下载文件无非要读取文件然后写文件，主要这两个步骤，主要难点：
1.读文件，就是硬盘到内存的过程，由于jdk内存限制，不能读的太大。
2.写文件，就是响应到浏览器端的过程，http协议是短链接，如果写文件太慢，时间过久，会造成浏览器死掉。
知识点：
1.org.apache.http.impl.client.CloseableHttpClient 模拟httpClient客户端发送http请求，可以控制到请求文件的字节位置。
2.BufferedInputStream都熟悉，用它接受请求来的流信息缓存。
3.RandomAccessFile文件随机类，可以向文件写入指定位置的流信息。
基于以上信息，我的实现思路就是首先判断下载文件大小，配合多线程分割定制http请求数量和请求内容，响应到写入到RandomAccessFile指定位置中。在俗点就是大的http分割成一个个小的http请求，相当于每次请求一个网页。
废话不说，上代码。
DownLoadManagerTest类：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
package xxxx;import java.io.File;import java.io.IOException;import java.io.RandomAccessFile;import java.net.HttpURLConnection;import java.net.URL;import java.util.concurrent.CountDownLatch;import org.apache.commons.lang.exception.ExceptionUtils;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;import org.junit.After;import org.junit.Before;import org.junit.Test;import org.junit.runner.RunWith;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.springframework.beans.factory.annotation.Autowired;import org.springframework.core.task.TaskExecutor;import org.springframework.test.context.ActiveProfiles;import org.springframework.test.context.ContextConfiguration;import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests;import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;/** * * 文件下载管理类 */@RunWith(SpringJUnit4ClassRunner.class)@ActiveProfiles("test")@ContextConfiguration(locations={"classpath:test/applicationContext.xml"})public class DownLoadManagerTest extends AbstractTransactionalJUnit4SpringContextTests{ private static final Logger LOGGER = LoggerFactory.getLogger(DownLoadManagerTest.class); /** * * 每个线程下载的字节数 */ private long unitSize = 1000 * 1024; @Autowired private TaskExecutor taskExecutor; private CloseableHttpClient httpClient; private Long starttimes; private Long endtimes; @Before public void setUp() throws Exception { starttimes = System.currentTimeMillis(); System.out.println("测试开始...."); } @After public void tearDown() throws Exception { endtimes = System.currentTimeMillis(); System.out.println("测试结束!!"); System.out.println("********************"); System.out.println("下载总耗时:"+(endtimes-starttimes)/1000+"s"); System.out.println("********************"); } public DownLoadManagerTest() { System.out.println("初始化测试类...."); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); cm.setMaxTotal(100); httpClient = HttpClients.custom().setConnectionManager(cm).build(); } /** * * 启动多个线程下载文件 */ @Test public void doDownload() throws IOException { String remoteFileUrl="http://{host}:{port}/{project}/xx.xml"; String localPath="E://test//"; String fileName = new URL(remoteFileUrl).getFile(); System.out.println("远程文件名称："+fileName); fileName = fileName.substring(fileName.lastIndexOf("/") + 1, fileName.length()).replace("%20", " "); System.out.println("本地文件名称："+fileName); long fileSize = this.getRemoteFileSize(remoteFileUrl); this.createFile(localPath+System.currentTimeMillis()+fileName, fileSize); Long threadCount = (fileSize/unitSize)+(fileSize % unitSize!=0?1:0); long offset = 0; CountDownLatch end = new CountDownLatch(threadCount.intValue()); if (fileSize <= unitSize) {// 如果远程文件尺寸小于等于unitSize DownloadThreadTest downloadThread = new DownloadThreadTest(remoteFileUrl, localPath+fileName, offset, fileSize,end,httpClient); taskExecutor.execute(downloadThread); } else {// 如果远程文件尺寸大于unitSize for (int i = 1; i < threadCount; i++) { DownloadThreadTest downloadThread = new DownloadThreadTest( remoteFileUrl, localPath+fileName, offset, unitSize,end,httpClient); taskExecutor.execute(downloadThread); offset = offset + unitSize; } if (fileSize % unitSize != 0) {// 如果不能整除，则需要再创建一个线程下载剩余字节 DownloadThreadTest downloadThread = new DownloadThreadTest(remoteFileUrl, localPath+fileName, offset, fileSize - unitSize * (threadCount-1),end,httpClient); taskExecutor.execute(downloadThread); } } try { end.await(); } catch (InterruptedException e) { LOGGER.error("DownLoadManager exception msg:{}",ExceptionUtils.getFullStackTrace(e)); e.printStackTrace(); }//
System.out.println("111111"); LOGGER.debug("下载完成！{} ",localPath+fileName); //return localPath+fileName; } /** * * 获取远程文件尺寸 */ private long getRemoteFileSize(String remoteFileUrl) throws IOException { long fileSize = 0; HttpURLConnection httpConnection = (HttpURLConnection) new URL( remoteFileUrl).openConnection(); httpConnection.setRequestMethod("HEAD"); int responseCode = httpConnection.getResponseCode(); if (responseCode >= 400) { LOGGER.debug("Web服务器响应错误!"); return 0; } String sHeader; for (int i = 1;; i++) { sHeader = httpConnection.getHeaderFieldKey(i); if (sHeader != null && sHeader.equals("Content-Length")) { System.out.println("文件大小ContentLength:" + httpConnection.getContentLength()); fileSize = Long.parseLong(httpConnection .getHeaderField(sHeader)); break; } } return fileSize; } /** * * 创建指定大小的文件 */ private void createFile(String fileName, long fileSize) throws IOException { File newFile = new File(fileName); RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); raf.setLength(fileSize); raf.close(); } public TaskExecutor getTaskExecutor() { return taskExecutor; } public void setTaskExecutor(TaskExecutor taskExecutor) { this.taskExecutor = taskExecutor; } } |
DownloadThreadTest类：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
package xxxx;import java.io.BufferedInputStream;import java.io.File;import java.io.IOException;import java.io.RandomAccessFile;import java.util.concurrent.CountDownLatch;import org.apache.commons.lang.exception.ExceptionUtils;import org.apache.http.client.ClientProtocolException;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.protocol.BasicHttpContext;import org.apache.http.protocol.HttpContext;import org.slf4j.Logger;import org.slf4j.LoggerFactory;/** * * 负责文件下载的类 */public class DownloadThreadTest extends Thread { private static final Logger LOGGER = LoggerFactory .getLogger(DownloadThreadTest.class); /** * * 待下载的文件 */ private String url = null; /** * * 本地文件名 */ private String fileName = null; /** * * 偏移量 */ private long offset = 0; /** * * 分配给本线程的下载字节数 */ private long length = 0; private CountDownLatch end; private CloseableHttpClient httpClient; private HttpContext context; /** * * @param url * 下载文件地址 * * @param fileName * 另存文件名 * * @param offset * 本线程下载偏移量 * * @param length * 本线程下载长度 * * * * @author Angus.wang * * */ public DownloadThreadTest(String url, String file, long offset, long length, CountDownLatch end, CloseableHttpClient httpClient) { this.url = url; this.fileName = file; this.offset = offset; this.length = length; this.end = end; this.httpClient = httpClient; this.context = new BasicHttpContext(); LOGGER.debug("偏移量=" + offset + ";字节数=" + length); } public void run() { try { HttpGet httpGet = new HttpGet(this.url); httpGet.addHeader("Range", "bytes=" + this.offset + "-" + (this.offset + this.length - 1)); CloseableHttpResponse response = httpClient.execute(httpGet, context); ; BufferedInputStream bis = new BufferedInputStream(response .getEntity().getContent()); byte[] buff = new byte[1024]; int bytesRead; File newFile = new File(fileName); RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); while ((bytesRead = bis.read(buff, 0, buff.length)) != -1) { raf.seek(this.offset); raf.write(buff, 0, bytesRead); this.offset = this.offset + bytesRead; } raf.close(); bis.close(); } catch (ClientProtocolException e) { LOGGER.error("DownloadThread exception msg:{}",ExceptionUtils.getFullStackTrace(e)); } catch (IOException e) { LOGGER.error("DownloadThread exception msg:{}",ExceptionUtils.getFullStackTrace(e)); } finally { end.countDown(); LOGGER.info(end.getCount() + " is go on!"); System.out.println(end.getCount() + " is go on!"); } }} |
application.xml
|
1
2
3
4
5
6
7
8
9
10
11
12
13
|
<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor"> <!-- 线程池活跃的线程数 --> <property name="corePoolSize" value="5" /> <!-- 线程池最大活跃的线程数 --> <property name="maxPoolSize" value="10" /> <!-- 队列的最大容量 --> <property name="queueCapacity" value="600" /> </bean> <bean id="downLoadManager" class="xx.DownLoadManagerTest"> <property name="taskExecutor" ref="taskExecutor" /> </bean> |
测试运行，500M,我这网速得半个小时左右。要想下载更大的文件，只要jdk内存够大，就无限更改队列最大容量吧。
如果不同意见，欢迎各位大神指正。
尊重原创，多谢作者分享，转载自：http://my.oschina.net/zmf/blog/336961
