Python Telethon 多处理实现

Question

我需要从电报群下载许多文件（超过1 GB），使用电视马拉松和单次通话下载很慢...（大约5、6 Mbps）我已经安装了cryptg但没有更改...

非常感谢 斯特凡诺

这是我尝试使用多处理并同时下载许多文件的示例代码，但不起作用......有什么建议吗？

from telethon import TelegramClient
from multiprocessing import Pool
from datetime import datetime
import time
import threading
import os


# Remember to use your own values from my.telegram.org!
api_id = HiddenByME
api_hash = 'HiddenByME'
bot_token= 'HiddenByME'

client = TelegramClient('anon', api_id, api_hash)
bot= TelegramClient('bot', api_id, api_hash).start(bot_token=bot_token)

movieschanneltofix = HiddenByME
movieschannelfixed = HiddenByME

downloadpath = "media/"

# Printing download progress
def callbackdownload(current, total):
    global start_time
    global end_time
    delta = (datetime.now()- start_time).total_seconds()
    bytes_remaining = total - current
    speed = round(((current / 1024) / 1024) / delta, 2)
    seconds_left = round(((bytes_remaining / 1024) / 1024) / float(speed), 2)
    print('Downloaded', round(current/1000000,2), 'out of', round(total/1000000,2),'Megabytes {:.2%}'.format(current / total), "->", "Trascorso:", round(delta/60, 2), " Minuti * Rimanente:", int(seconds_left), "Secondi * Velocità:", round(speed, 2), "Mbps",  end='\r')
# Printing Upload Progress
def callbackupload(current, total):
    global start_time
    global end_time
    delta = (datetime.now()- start_time).total_seconds()
    bytes_remaining = total - current
    speed = round(((current / 1024) / 1024) / delta, 2)
    seconds_left = round(((bytes_remaining / 1024) / 1024) / float(speed), 2)
    print('Uploaded', round(current/1000000,2), 'out of', round(total/1000000,2),'Megabytes {:.2%}'.format(current / total), "->", "Trascorso:", round(delta/60, 2), " Minuti * Rimanente:", int(seconds_left), "Secondi * Velocità:", round(speed, 2), "Mbps",  end='\r')


async def moviefix(results):
    global end_time
    global start_time
    print("Fixing Message ID: ", results.split(';')[0], "\n", "OldName: ", results.split(';')[1], "\n", "NewName: ", results.split(';')[2])

    # Get Message and Download Media Renaming It
    downpath = downloadpath+results.split(';')[2]
    start_time = datetime.now()
    print("Download Avviato:", start_time.strftime("%d/%m/%Y, %H:%M:%S"))
    path = await client.download_media(await client.get_messages(movieschanneltofix, ids=int(results.split(';')[0])), downpath, progress_callback=callbackdownload)
    print('Download Completato!!! ', path)
    time.sleep(1)

    #Upload Renamed File
    start_time = datetime.now()
    print("Upload Avviato:", start_time.strftime("%d/%m/%Y, %H:%M:%S"))
    result = await bot.send_file(movieschannelfixed, downpath, force_document=True, progress_callback=callbackupload)
    print('Upload Completato!!! ', result.file.name)
    time.sleep(1)

    #Delete Uploaded File
    os.remove(downpath)
    time.sleep(1)

    #Save Fixed File in DB
    with open('data/lastmoviefixed.txt', 'a') as f:
        f.write(results+"\n")
    time.sleep(1)
    time.sleep(20)
    return results


#    print (results, "PID:", os.getpid())
#    with open('data/lastmoviefixed.txt', 'a') as f:
#        f.write(results+"\n")
#        time.sleep(1)
#    time.sleep(10)
#    return results

if __name__ == "__main__":
    global listtodo
    listtodo = set(open('data/listtofix.txt', 'r', encoding="utf8").read().splitlines()) - set(open('data/lastmoviefixed.txt', encoding="utf8").read().splitlines())
    with Pool(1) as p:
        result = p.map_async(client.loop.run_until_complete(moviefix), sorted(listtodo))
        while not result.ready():
            time.sleep(1)
        result = result.get()
        p.terminate()
        p.join()
    print("Done", results)

Answer 1

您应该对 CPU 密集型操作使用多处理。对于 I/O 密集型操作，您可以使用 async.io 并发运行请求。然而，当网络带宽不是限制因素时（例如发送许多请求时），并发性是最有利的。在下载文件的情况下，如果您达到最大网络带宽，您将从并发中获得的好处最小。