Created
May 6, 2021 17:54
-
-
Save zfhxi/9dfa98c4328e3887938e6a6956ac23c5 to your computer and use it in GitHub Desktop.
给定多个包的字符串从https://archive.archlinux.org/packages下载
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import re | |
| import os | |
| import requests | |
| from requests.adapters import HTTPAdapter | |
| from requests.packages.urllib3.util.retry import Retry | |
| from time import time | |
| from multiprocessing.dummy import Pool | |
| #下载文件参考 https://cloud.tencent.com/developer/article/1471279 | |
| arch_mirrors_url='https://archive.archlinux.org/packages' | |
| pkgs_str=''' | |
| audacious-plugins-4.1-3 confuse-3.3-3 faad2-2.10.0-1 fluidsynth-2.1.8-1 libbs2b-3.1.0-7 | |
| libftdi-1.5-2 libguess-1.2-3 libinstpatch-1.1.6-1 libmms-0.6.4-3 libmtp-1.1.18-1 | |
| libsidplayfp-2.1.1-1 libusb-compat-0.1.7-1 lirc-1:0.10.1-8 mpg123-1.26.5-1 neon-0.31.2-1 | |
| audacious-4.1-2 | |
| ''' | |
| save_path='/home/cszmx/Downloads/pkgs' | |
| proxies = { | |
| "http": "http://127.0.0.1:1082", | |
| "https": "http://127.0.0.1:1082", | |
| } | |
| if not os.path.exists(save_path): | |
| os.mkdir(save_path) | |
| ############################## code begin ############################## | |
| pkgs=list(filter(None,re.split(r'\s+',pkgs_str) )) | |
| def fill_url(x): | |
| ''' | |
| 拼接url | |
| :param x: | |
| :return: | |
| ''' | |
| first_char=x[0] | |
| pkg_name=re.search(r'([a-z][a-z\-]*[a-z][0-9]?)',x).group(0) | |
| return f'{arch_mirrors_url}/{first_char}/{pkg_name}/{x}-x86_64.pkg.tar.zst' | |
| pkgs_urls=list(map(fill_url,pkgs)) | |
| pkgs_paths=list(map(lambda x:os.path.join(save_path,f'{x}-x86_64.pkg.tar.zst'),pkgs)) | |
| path_pkgs_urls=list(zip(pkgs_paths,pkgs_urls)) | |
| for path_pkg in path_pkgs_urls: | |
| print(path_pkg) | |
| print(f'\n') | |
| def download_pkg(url): | |
| session = requests.Session() | |
| retry = Retry(connect=3, backoff_factor=0.5) | |
| adapter = HTTPAdapter(max_retries=retry) | |
| session.mount('http://', adapter) | |
| session.mount('https://', adapter) | |
| path, url = url | |
| try: | |
| r = session.get(url, stream=True,proxies=proxies) | |
| with open(path, 'wb') as f: | |
| for ch in r: | |
| f.write(ch) | |
| print(f'[OK]: {path} has been finished!') | |
| except Exception as e: | |
| print(f'[ERROR]: Failed to download {url}') | |
| error_log_path=os.path.join(save_path,'error_urls.log') | |
| with open(error_log_path,'wb') as f: | |
| f.write(f'{path},{url}') | |
| start = time() | |
| pool=Pool(len(path_pkgs_urls)) | |
| result=pool.map(download_pkg,path_pkgs_urls) | |
| pool.close() | |
| pool.join() | |
| # for x in path_pkgs_urls: | |
| # url_response(x) | |
| print(f"Time to download: {time() - start}") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment