python之试图将文件下载缓冲区拆分为单独的线程

我正在尝试将文件缓冲区下载到 5 个线程中，但它似乎出现了乱码。

from numpy import arange 
import requests 
from threading import Thread 
import urllib2 
 
url = 'http://pymotw.com/2/urllib/index.html' 
sizeInBytes = r = requests.head(url, headers={'Accept-Encoding': 'identity'}).headers['content-length'] 
 
splitBy = 5 
 
splits = arange(splitBy + 1) * (float(sizeInBytes)/splitBy) 
 
dataLst = [] 
 
def bufferSplit(url, idx, splits): 
    req = urllib2.Request(url,  headers={'Range': 'bytes=%d-%d' % (splits[idx], splits[idx+1])}) 
    print {'bytes=%d-%d' % (splits[idx], splits[idx+1])} 
    dataLst.append(urllib2.urlopen(req).read()) 
 
 
for idx in range(splitBy): 
    dlth = Thread(target=bufferSplit, args=(url, idx, splits)) 
    dlth.start() 
 
 
print dataLst 
 
with open('page.html', 'w') as fh: 
    fh.write(''.join(dataLst)) 

更新: 所以我努力工作，但进展甚微，但是，如果我下载一个 jpg，它似乎已损坏；

from numpy import arange 
import os 
import requests 
import threading 
import urllib2 
 
# url ='http://s1.fans.ge/mp3/201109/08/John_Legend_So_High_Remix(fans_ge).mp3' 
url = "http://www.nasa.gov/images/content/607800main_kepler1200_1600-1200.jpg" 
# url = 'http://pymotw.com/2/urllib/index.html' 
sizeInBytes = requests.head(url, headers={'Accept-Encoding': 'identity'}).headers.get('content-length', None) 
 
 
splitBy = 5 
 
dataLst = [] 
 
 
class ThreadedFetch(threading.Thread): 
    """ docstring for ThreadedFetch 
    """ 
    def __init__(self, url, fileName, splitBy=5): 
        super(ThreadedFetch, self).__init__() 
        self.__url = url 
        self.__spl = splitBy 
        self.__dataLst = [] 
        self.__fileName = fileName 
 
    def run(self): 
        if not sizeInBytes: 
            print "Size cannot be determined." 
            return 
        splits = arange(self.__spl + 1) * (float(sizeInBytes)/self.__spl) 
        for idx in range(self.__spl): 
            req = urllib2.Request(self.__url,  headers={'Range': 'bytes=%d-%d' % (splits[idx], splits[idx+1])}) 
            self.__dataLst.append(urllib2.urlopen(req).read()) 
 
 
    def getFileData(self): 
        return ''.join(self.__dataLst) 
 
 
fileName = url.split('/')[-1] 
 
dl = ThreadedFetch(url, fileName) 
dl.start() 
dl.join() 
content = dl.getFileData() 
if content: 
    with open(fileName, 'w') as fh: 
        fh.write(content) 
    print "Finished Writing file %s" % fileName 

下面是下载后的图片。

请您参考如下方法:

这是该项目的另一个版本。差异:

• 线程代码是一个单独的小函数

• 每个线程下载一个 block ，然后将其存储在全局线程安全字典中

• 线程启动，然后 join()ed -- 它们同时运行

• 全部完成后，数据以正确的顺序重组，然后写入磁盘

• 额外打印，以验证一切正确

• 计算输出文件大小，用于额外比较

来源

import os, requests 
import threading 
import urllib2 
import time 
 
URL = "http://www.nasa.gov/images/content/607800main_kepler1200_1600-1200.jpg" 
 
def buildRange(value, numsplits): 
    lst = [] 
    for i in range(numsplits): 
        if i == 0: 
            lst.append('%s-%s' % (i, int(round(1 + i * value/(numsplits*1.0) + value/(numsplits*1.0)-1, 0)))) 
        else: 
            lst.append('%s-%s' % (int(round(1 + i * value/(numsplits*1.0),0)), int(round(1 + i * value/(numsplits*1.0) + value/(numsplits*1.0)-1, 0)))) 
    return lst 
 
def main(url=None, splitBy=3): 
    start_time = time.time() 
    if not url: 
        print "Please Enter some url to begin download." 
        return 
 
    fileName = url.split('/')[-1] 
    sizeInBytes = requests.head(url, headers={'Accept-Encoding': 'identity'}).headers.get('content-length', None) 
    print "%s bytes to download." % sizeInBytes 
    if not sizeInBytes: 
        print "Size cannot be determined." 
        return 
 
    dataDict = {} 
 
    # split total num bytes into ranges 
    ranges = buildRange(int(sizeInBytes), splitBy) 
 
    def downloadChunk(idx, irange): 
        req = urllib2.Request(url) 
        req.headers['Range'] = 'bytes={}'.format(irange) 
        dataDict[idx] = urllib2.urlopen(req).read() 
 
    # create one downloading thread per chunk 
    downloaders = [ 
        threading.Thread( 
            target=downloadChunk,  
            args=(idx, irange), 
        ) 
        for idx,irange in enumerate(ranges) 
        ] 
 
    # start threads, let run in parallel, wait for all to finish 
    for th in downloaders: 
        th.start() 
    for th in downloaders: 
        th.join() 
 
    print 'done: got {} chunks, total {} bytes'.format( 
        len(dataDict), sum( ( 
            len(chunk) for chunk in dataDict.values() 
        ) ) 
    ) 
 
    print "--- %s seconds ---" % str(time.time() - start_time) 
 
    if os.path.exists(fileName): 
        os.remove(fileName) 
    # reassemble file in correct order 
    with open(fileName, 'w') as fh: 
        for _idx,chunk in sorted(dataDict.iteritems()): 
            fh.write(chunk) 
 
    print "Finished Writing file %s" % fileName 
    print 'file size {} bytes'.format(os.path.getsize(fileName)) 
 
if __name__ == '__main__': 
    main(URL) 

输出

102331 bytes to download. 
done: got 3 chunks, total 102331 bytes 
--- 0.380599021912 seconds --- 
Finished Writing file 607800main_kepler1200_1600-1200.jpg 
file size 102331 bytes