来一个python 下载模块 支持多任务有回调 有进度 支持同步 ...

已注销41434

import os
import requests
import threading
import time


class Downloader:
    def __init__(
        self,
        id,
        completion_callback=None,
        progress_callback=None,
        message_output=None,
        download_dir="cache",
    ):
        self.id = id
        self.file_urls = []
        self.completion_callback = completion_callback
        self.progress_callback = progress_callback
        self.message_output = message_output
        self.download_dir = download_dir
        self.downloaded_files = []
        self.stop_flag = False
        self.session = requests.Session()
        self._create_download_dir()

    def _create_download_dir(self):
        os.makedirs(self.download_dir, exist_ok=True)

    def add_url(self, url, filename, dirname=None):
        self.file_urls.append((url, filename, dirname))

    def start(self, callback=None, params=None):
        download_thread = threading.Thread(
            target=self._download,
            args=(
                callback,
                params,
            ),
        )
        download_thread.start()

    def start_sync(self, callback=None, params=None):
        return self._download(callback, params)

    def stop(self):
        self.stop_flag = True

    def _download(self, callback=None, params=None):
        for url, filename, dirname in self.file_urls:
            if self.stop_flag:
                break
            try:
                self._download_file(url, filename, dirname)
            except Exception as e:
                self._print_message(f"Failed to download {url}: {e}")
                break

        if self.completion_callback:
            sync = self.completion_callback(
                self.id, self.downloaded_files, self.download_dir
            )
            if callback:
                return callback(self.id, sync, params)
        self.session.close()

    def _download_file(self, url, filename, dirname):
        response = self.session.get(url, stream=True)

        total_size = int(response.headers.get("content-length", 0))
        bytes_so_far = 0
        start_time = time.time()

        # Remove trailing slash from self.download_dir if present
        download_dir = self.download_dir.rstrip(os.path.sep)

        # Concatenate paths while ensuring dirname starts with a directory separator
        path = os.path.join(download_dir, dirname.lstrip(os.path.sep) if dirname else "")
        os.makedirs(path, exist_ok=True)

        file_path = os.path.join(path, filename)
        with open(file_path, "wb") as file:
            for chunk in response.iter_content(chunk_size=1024):
                if self.stop_flag:
                    break
                if chunk:
                    file.write(chunk)
                    bytes_so_far += len(chunk)
                    if time.time() - start_time > 1:
                        if self.progress_callback:
                            self.progress_callback(
                                self.id, bytes_so_far, total_size, filename, dirname
                            )
                        start_time = time.time()
        self._print_message(f"{os.path.join(dirname or '', filename)} downloaded from {url}")
        self.downloaded_files.append(file_path)

    def _print_message(self, message):
        if self.message_output:
            self.message_output(message)
        else:
            print(message)


if __name__ == "__main__":

    def handle_download_completion(id, downloaded_files, download_dir):
        print_message(f"All files have been downloaded for download ID: {id}")
        print_message(f"Downloaded files: {downloaded_files}")
        return True

    def handle_progress_update(id, bytes_so_far, total_size, filename, dirname):
        print_message(
            f"Download ID: {id}, downloaded {bytes_so_far} of {total_size} bytes for {dirname} {filename}"
        )

    def print_message(message):
        print(message)

    base = os.path.dirname(__file__)
    cache = os.path.join(base, "static", "cache")
    downloader1 = Downloader(
        22, handle_download_completion, handle_progress_update, print_message, cache
    )

    downloader1.add_url("https://dldir1.qq.com/qqfile/qq/QQNT/a8b6c3ae/QQ9.9.6.20201_x64.exe", "QQSetup.exe")
    def callback(id, state, value):
        print("#callback", id, state,value)
    print(downloader1.start(callback,'is'))




补充内容 (2024-5-15 09:42):
更新了一下_download 下载路径处理规则 并自动处理路径转义 见10楼

已注销41434

import os
import requests
import threading
import time


class Downloader:
    def __init__(
        self,
        id,
        completion_callback=None,
        progress_callback=None,
        message_output=None,
        download_dir="cache",
    ):
        self.id = id
        self.file_urls = []
        self.completion_callback = completion_callback
        self.progress_callback = progress_callback
        self.message_output = message_output
        self.download_dir = download_dir
        self.downloaded_files = []
        self.stop_flag = False
        self.session = requests.Session()
        self._create_download_dir()

    def _create_download_dir(self):
        os.makedirs(self.download_dir, exist_ok=True)

    def add_url(self, url, filename, dirname=None):
        self.file_urls.append((url, filename, dirname))

    def start(self, callback=None, params=None):
        download_thread = threading.Thread(
            target=self._download,
            args=(
                callback,
                params,
            ),
        )
        download_thread.start()

    def start_sync(self, callback=None, params=None):
        return self._download(callback, params)

    def stop(self):
        self.stop_flag = True

    def _download(self, callback=None, params=None):
        for url, filename, dirname in self.file_urls:
            if self.stop_flag:
                break
            try:
                self._download_file(url, filename, dirname)
            except Exception as e:
                self._print_message(f"Failed to download {url}: {e}")
                break

        if self.completion_callback:
            sync = self.completion_callback(
                self.id, self.downloaded_files, self.download_dir
            )
        self.session.close()
        if callback:
            return callback(self.id, sync, params)

    def _download_file(self, url, filename, dirname):
        response = self.session.get(url, stream=True)
        total_size = int(response.headers.get("content-length", 0))
        bytes_so_far = 0
        start_time = time.time()
        file_path = os.path.join(self.download_dir, dirname or "", filename)
        file_path = os.path.normpath(file_path)
        os.makedirs(os.path.dirname(file_path), exist_ok=True)
        with open(file_path, "wb") as file:
            for chunk in response.iter_content(chunk_size=1024):
                if self.stop_flag:
                    break
                if chunk:
                    file.write(chunk)
                    bytes_so_far += len(chunk)
                    if time.time() - start_time > 1:
                        if self.progress_callback:
                            self.progress_callback(
                                self.id, bytes_so_far, total_size, filename, dirname
                            )
                        start_time = time.time()
        self._print_message(
            f"{os.path.join(dirname or '', filename)} downloaded from {url}"
        )
        self.downloaded_files.append(file_path)

    def _print_message(self, message):
        if self.message_output:
            self.message_output(message)
        else:
            print(message)


if __name__ == "__main__":

    def handle_download_completion(id, downloaded_files, download_dir):
        print_message(f"All files have been downloaded for download ID: {id}")
        print_message(f"Downloaded files: {downloaded_files}")
        return True

    def handle_progress_update(id, bytes_so_far, total_size, filename, dirname):
        print_message(
            f"Download ID: {id}, downloaded {bytes_so_far} of {total_size} bytes for {dirname} {filename}"
        )

    def print_message(message):
        print(message)

    base = os.path.dirname(__file__)
    cache = os.path.join(base, "static", "cache")
    downloader1 = Downloader(
        22, handle_download_completion, handle_progress_update, print_message, cache
    )
    downloader1.add_url(
        "http://192.168.0.201:8998/html/data/download/schedule-88.zip",
        "os\\schedule-1.zip",
    )

    # downloader1.add_url("https://dldir1.qq.com/qqfile/qq/QQNT/a8b6c3ae/QQ9.9.6.20201_x64.exe", "QQSetup.exe")
    def callback(id, state, value):
        print("#callback", id, state, value)

    print(downloader1.start(callback, "is"))

shuya1

感谢分享，很给力！~

光影魔术

感谢分享源码

已注销41434

lisher 发表于 2024-5-13 12:41
您的代码中存在一些问题，这可能会影响其在实际运行中的表现：

1.print_message函数在Downloader类中被定 ...

使用方法没更新 不过不影响吃饭

一指温柔

感谢分享，很给力！~

bianyuan456

已经顶贴，感谢您对论坛的支持！

一指温柔

支持开源~！感谢分享

shinbudar

新技能已get√

ZHuanR

新技能已get√