Skip to main content
 首页 » 编程设计

内存使用

2025年05月04日32xing901022

所以我有一些代码可以接收一组文件,将它可以拼接在一起,然后绘制它们。 我发布了我的大部分代码,试图使它更具可读性,如果需要可以添加更多

for paths,dirs,files in os.walk(start_path): 
for d in dirs: 
    path = start_path +  changedir + d 
    pathpart = d 
 
    os.chdir(path) 
    for file in glob.glob("*-0.dat"): 
        tempname = file.split("-") 
        fileName1 = str(tempname[0] + "-" + tempname[1]+ "-") 
        gc.collect() 
 
        Chan2 = [] 
        Chan1 = [] 
        temp_1 = [] 
        temp_2 = [] 
        temp_3 = [] 
        Data_Sets1 = [] 
        Data_Sets2 = [] 
        Headers = [] 
 
 
        for fileNumber in range(0,45): 
            fileName = fileName1 + str(fileNumber) + fileName3 
            header, data1, data2 = u.unpackFile(path,fileName) 
 
            if header == None: 
                logging.warning("curropted file found at " + fileName) 
                Data_Sets1.append(temp_1) 
                Data_Sets2.append(temp_2) 
                Headers.append(temp_3) 
                temp_1 = [] 
                temp_2 = [] 
                temp_3 = [] 
            else: 
                logging.info(fileName + " is good!") 
                temp_3.append(header) 
                for i in range(0,10000): 
                    temp_1.append(data1[i]) 
                    temp_2.append(data2[i]) 
 
        Data_Sets1.append(temp_1) 
        Data_Sets2.append(temp_2) 
        Headers.append(temp_3) 
        temp_1 = [] 
        temp_2 = [] 
        temp_3 = [] 
 
        del temp_1 
        del temp_2 
        del temp_3 
 
        lengths = [] 
        for i in range(len(Data_Sets1)): 
            lengths.append(len(Data_Sets1[i])) 
        index = lengths.index(max(lengths)) 
 
        Chan1 = Data_Sets1[index] 
        Chan2 = Data_Sets2[index] 
        Start_Header = Headers[index] 
        if (len(Chan1) == 0 | len(Chan2) == 0): 
            continue 
        try: 
            Date = Start_Header[index][0] 
            Time = Start_Header[index][1] 
        except IndexError: 
            logging.critical("file " + fileName + " is unusuable") 
            continue 
        """ 
        Clostest_Power = int(np.log(len(Chan1))/np.log(2)) 
        Length = 2 ** Clostest_Power 
        logging.debug("Length of the file is " + str(Length)) 
        Chan1 = Chan1[0:Length] 
        Chan2 = Chan2[0:Length] 
        """ 
        logging.debug("Length of channels is " + str(len(Chan1)))    
 
        window = np.hanning(Window_Width) 
 
        t= s.Time_Array(len(Chan1),Sample_Rate) 
 
 
        window2 = np.hanning(len(Chan1)) 
 
        Noise_Frequincies = [] 
        for i in range(1,125): 
            Noise_Frequincies.append(60.0*float(i)) 
        Noise_Frequincies.append(180.0) 
 
        filter1 = s.Noise_Reduction(Sample_Rate,Noise_Frequincies,Chan1) 
        filter2 = s.Noise_Reduction(Sample_Rate,Noise_Frequincies,Chan2) 
 
        logging.info("Starting the plots") 
 
 
        fig1, (ax1, ax2) = plt.subplots(nrows=2) 
 
 
        spec1, freqs1, time1 = mlab.specgram(filter1, NFFT=Window_Width, Fs=Sample_Rate, window=window, noverlap=Over_Lap) 
 
 
        im1 = ax1.imshow(spec1, cmap=cm.get_cmap("rainbow"), norm=colors.LogNorm(), origin='lower', 
            extent=[t[0], t[len(t)-1], freqs1.min(), 8000],aspect='auto',vmin=1e-5,vmax=1e5) 
 
        ax1.set_title(str(Date) + "-" + str(Time) + " Channel 1") 
        ax1.set_ylabel("Freqency Hz") 
 
 
 
        spec2, freqs2, time2 = mlab.specgram(filter2, NFFT=Window_Width, Fs=Sample_Rate, window=window, noverlap=Over_Lap) 
 
        im2 = ax2.imshow(spec2, cmap=cm.get_cmap("rainbow"), norm=colors.LogNorm(), origin='lower', 
            extent=[t[0], t[len(t)-1], freqs2.min(), 8000],aspect='auto',vmin=1e-5,vmax=1e5) 
 
        cax1, kw1 = matplotlib.colorbar.make_axes(ax1) 
        colorbar(im1,cax=cax1,**kw1) 
        cax2, kw2 = matplotlib.colorbar.make_axes(ax2) 
        colorbar(im2,cax=cax2,**kw2) 
 
        ax2.set_title(str(Date) + "-" + str(Time) + " Channel 2") 
        ax2.set_ylabel("Freqency Hz") 
 
 
        save1 = save_path+pathpart + changedir+specgram_path 
        if not os.path.exists(save1): 
            os.makedirs(save1) 
        savefig(os.path.join(save1,str(Date) + "-" + str(Time) + "-Power_Spec1.png")) 
 
        logging.info("Spectrogram path is " + save1) 
 
 
 
 
        fig2, (ax4,ax6) = plt.subplots(nrows=2) 
        final_fft = [] 
        fft = s.Full_FFT(filter1,window2) 
        for i in range(0,len(fft)): 
            final_fft.append(np.absolute(fft[i])) 
 
 
 
        freqs = [] 
        for i in range(0,len(final_fft)): 
            freqs.append(i*Sample_Rate/float(len(final_fft))) 
 
        ax4.plot(freqs, final_fft) 
 
        new_fft = [] 
        new = s.Full_FFT(filter2,window2) 
        for i in range(0,len(new)): 
            new_fft.append(np.absolute(new[i])) 
        ax6.plot(freqs,new_fft) 
 
 
        save2 = save_path+pathpart+ changedir + freq_path 
        logging.info("Frequency path is " + save2) 
        if not os.path.exists(save2): 
            os.makedirs(save2) 
 
        savefig(os.path.join(save2,str(Date) + "-" + str(Time) + "-Freq.png")) 
 
        ax4.set_title(str(Date) + "-" + str(Time) +" Channel 1") 
        ax4.set_xlabel("Bins") 
        ax4.set_ylabel("Power") 
 
 
        ax6.set_title(str(Date) + "-" + str(Time) + " Channnel 2") 
        ax6.set_xlabel("Bins") 
        ax6.set_ylabel("Power") 
 
        fig3, (ax7, ax9) = plt.subplots(nrows=2) 
        ax7.plot(t,filter1) 
        ax9.plot(t,filter2) 
 
        save3 = save_path+pathpart + changedir +signal_path 
        if not os.path.exists(save3): 
            os.makedirs(save3) 
        savefig(os.path.join(save3,str(Date) + "-" + str(Time) + "-Signal.png")) 
 
        logging.info("Signal path is " + save3) 
 
        fig1.clf() 
        fig2.clf() 
        fig3.clf() 
        matplotlib.pyplot.clf() 
        close('all') 
        gc.collect() 

这是解压代码

def unpackFile(path,fileName): 
header = "" 
startKey = "" 
dataList = [] 
chan1 = [] 
chan2 = [] 
found = False 
logging.info("Starting file " + fileName) 
if not os.path.isfile(os.path.join(path,fileName)): 
    logging.warning("could not find "+fileName) 
    return None, None, None 
try: 
    contents = open(os.path.join(path,fileName),'rb') 
except IOError: 
    logging.warning(fileName + " Not found") 
    return None, None, None 
 
 
 
#looks for the closing bracket in the header of the file 
filesize = os.path.getsize(os.path.join(path,fileName)) 
if filesize < 1000: 
    logging.warning(fileName + " is below 1000 bytes") 
    contents.close() 
    contents = None 
    return None, None, None 
while found==False: 
    char = contents.read(1) 
    #print char 
    header = header + char 
    if char == "}": 
        #Once the close bracket is found, the next 10 characters should be the start key 
        startKey = contents.read(10) 
        #header = header + startKey 
        #print("found the }") 
        found = True  
if startKey=="Data_Start": 
    logging.info("Found start key for file "+fileName) 
else: 
    logging.warning("No start key found " + fileName + " is corrupted") 
    contents.close() 
    contents = None 
    return None, None, None    
#Looks for the end key in the file  
try: 
    logging.debug("Reading the data") 
    data = contents.read(40000) 
    #endKey = data[len(data)-10:len(data)] 
    endKey = contents.read() 
except IOError: 
    logging.warning("IOE error trying to read the end key") 
    endKey="" 
    contents.close() 
    contents = None 
    return None, None, None 
 
if endKey == "Data_Stop ": 
    logging.debug("Found end key " ) 
else: 
    logging.debug("No end key found in" +fileName) 
#Unpacks the data from binary into signed ints 
for i in range(0,len(data),2): 
    value = data[i:i+2] 
    if len(value)==2: 
        number = struct.unpack('>h',data[i:i+2]) 
        #print number 
        dataList.append(number[0]) 
    else: 
        break 
logging.debug("total points found is " + str(len(dataList))) 
#Splits data into two channels 
for j in range(0,len(dataList)): 
    if j%2==0: 
        chan2.append(dataList[j]) 
        #if dataList[j] != 0: 
            #print("chan2 has a non 0 " + str(j)) 
    else: 
        chan1.append(dataList[j]) 
#Checks to make sure both channels contain 10000 data points. If this is not true the file is curppted 
if len(chan2)!=10000: 
    logging.warning("Chanel 2 did not containg the right number of data points, " + fileName + " is corupted") 
    contents.close() 
    contents = None 
    return None, None, None 
if len(chan1)!=10000: 
    logging.warning("Chanel 1 did not containg the right number of data points, " + fileName + " is corupted") 
    contents.close() 
    contents = None 
    return None, None, None 
contents.close() 
contents = None 
 
header = header[1:len(header)-1] 
header_parts = header.split(',') 
return header_parts,chan1,chan2 

某处内存泄漏,我不知道在哪里。我试图让代码遍历目录,挑选出数据集,然后绘制它们。 几分钟后,这会吃掉几 GB 的内存。 有什么减少它们的技巧吗?

请您参考如下方法:

在像您这样的情况下,我更愿意将清理工作留给操作系统。因此,我将泄漏内存的代码作为一个单独的进程运行:

from multiprocessing import Process, Queue 
 
def memory_leaking_code(arg1, q): 
   """Your memory leaking code goes here""" 
   print arg1 
   q.put('data from memory_leaking_code()') 
 
def main(): 
   q = Queue() 
   p = Process(target=memory_leaking_code, args=('data to memory_leaking_code()', q)) 
   p.start() 
   print q.get() 
   if p.is_alive(): 
      p.terminate() 
 
main()